World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! Databricks SQL Statement Execution API(`/api/2.0/sql/statements`)。
//! 一个 SQL warehouse 后面是 Unity Catalog 的整个湖仓(Delta 为主),
//! 令牌是 personal access token。结果要 INLINE + JSON_ARRAY:值全是字符串,
//! 按 manifest 里的列类型还原成数与布尔。
use super::conn::DatabricksConn;
use super::{
coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult,
SchemaColumn, HTTP_POLL_BUDGET, ROW_CAP,
};
use serde::Deserialize;
use serde_json::json;
use std::time::{Duration, Instant};
pub struct DatabricksEngine {
conn: DatabricksConn,
}
#[derive(Deserialize)]
struct StatementResponse {
statement_id: Option<String>,
status: Status,
manifest: Option<Manifest>,
result: Option<ResultData>,
}
#[derive(Deserialize)]
struct Status {
state: String,
error: Option<StatusError>,
}
#[derive(Deserialize)]
struct StatusError {
message: Option<String>,
error_code: Option<String>,
}
#[derive(Deserialize)]
struct Manifest {
schema: Option<Schema>,
/// INLINE disposition with a `row_limit` set:true 表示结果按行截断。
/// schema 读不传 row_limit,理论上不会触发;留着做显式 fail-fast 而不是静默返回
/// 不全的数据
truncated: Option<bool>,
/// 整个结果集的总行数(chunked INLINE 时也有)。schema 读完后用来校验
/// `rows.len() == total_row_count`
total_row_count: Option<u64>,
}
#[derive(Deserialize, Clone)]
struct Schema {
columns: Vec<ColumnInfo>,
}
#[derive(Deserialize, Clone)]
struct ColumnInfo {
name: String,
type_text: Option<String>,
}
#[derive(Deserialize)]
struct ResultData {
data_array: Option<Vec<Vec<serde_json::Value>>>,
/// 首块在 StatementResponse.result 里,后续 GET 直接返回这个对象。
/// 链接由服务端给出,路径与查询串都要原样带到下一次请求。
next_chunk_internal_link: Option<String>,
}
impl DatabricksEngine {
pub fn new(conn: DatabricksConn) -> Self {
Self { conn }
}
/// information_schema.columns 的候选写法 `(说明, SQL)`,说明只用来报错。
/// 没有 catalog 时只剩一条:会话默认那份。
fn schema_queries(&self) -> Vec<(&'static str, String)> {
let schema_filter = self
.conn
.schema
.as_deref()
.map(|s| format!(" AND table_schema = {}", sql_literal(s)))
.unwrap_or_default();
let select = "SELECT table_schema, table_name, column_name, data_type, comment";
let tail = format!(
"table_schema <> 'information_schema'{schema_filter} \
ORDER BY table_schema, table_name, ordinal_position"
);
match self.conn.catalog.as_deref() {
Some(catalog) => vec![
(
"catalog information_schema",
format!(
"{select} FROM `{}`.information_schema.columns WHERE {tail}",
catalog.replace('`', "``")
),
),
(
"system information_schema",
format!(
"{select} FROM system.information_schema.columns \
WHERE table_catalog = {} AND {tail}",
sql_literal(catalog)
),
),
],
None => vec![(
"session information_schema",
format!("{select} FROM information_schema.columns WHERE {tail}"),
)],
}
}
/// 提交一条语句,轮询到 SUCCEEDED,把结果拼成 (列名, 行值)。所有列值都按
/// manifest 的 `type_text` 还原成数或布尔。
///
/// `row_limit`:传 `None` 表示不限行(schema 读用)。INLINE 的 JSON_ARRAY
/// 结果可能分成多块;首块之后通过 `next_chunk_internal_link`
/// 继续 GET,循环到 result 没有给出下一页为止。schema 读完后用
/// `manifest.total_row_count` 做一次 `rows.len()` 校验——少了就当结果不全
/// (docs 说 truncated=true 时另有标记,schema 读不该见到)
///
/// 之前对所有语句都传 `row_limit: ROW_CAP + 1`(#703):chat 查询需要这个
/// 安全阀,schema 读不需要;information_schema.columns 每行是一列,宽 catalog
/// 上的 schema 文档会被悄悄截到 201 行
async fn run(
&self,
sql: &str,
row_limit: Option<usize>,
) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
let client = super::http()?;
let mut body = json!({
"warehouse_id": self.conn.warehouse_id,
"statement": sql,
"wait_timeout": "30s",
"on_wait_timeout": "CONTINUE",
"disposition": "INLINE",
"format": "JSON_ARRAY",
});
if let Some(n) = row_limit {
body["row_limit"] = json!(n);
}
if let Some(c) = &self.conn.catalog {
body["catalog"] = json!(c);
}
if let Some(s) = &self.conn.schema {
body["schema"] = json!(s);
}
let mut resp: StatementResponse = client
.post(format!("{}/api/2.0/sql/statements", self.conn.base))
.bearer_auth(&self.conn.token)
.json(&body)
.send()
.await?
.error_for_status()?
.json()
.await?;
let started = Instant::now();
loop {
match resp.status.state.as_str() {
"SUCCEEDED" => break,
"PENDING" | "RUNNING" => {
let id = resp
.statement_id
.clone()
.ok_or_else(|| anyhow::anyhow!("Databricks returned no statement_id"))?;
if started.elapsed() > HTTP_POLL_BUDGET {
anyhow::bail!(
"Databricks statement did not finish within {}s",
HTTP_POLL_BUDGET.as_secs()
);
}
tokio::time::sleep(Duration::from_secs(1)).await;
resp = client
.get(format!("{}/api/2.0/sql/statements/{id}", self.conn.base))
.bearer_auth(&self.conn.token)
.send()
.await?
.error_for_status()?
.json()
.await?;
}
other => {
let e = resp.status.error.as_ref();
let code = e
.and_then(|e| e.error_code.clone())
.map(|c| format!("{c}: "))
.unwrap_or_default();
let msg = e
.and_then(|e| e.message.clone())
.unwrap_or_else(|| format!("statement ended in state {other}"));
anyhow::bail!("{code}{msg}");
}
}
}
let columns: Vec<ColumnInfo> = resp
.manifest
.as_ref()
.and_then(|m| m.schema.as_ref())
.map(|s| s.columns.clone())
.unwrap_or_default();
let mut rows: Vec<Vec<serde_json::Value>> = resp
.result
.as_ref()
.and_then(|r| r.data_array.clone())
.unwrap_or_default()
.into_iter()
.map(|row| {
row.iter()
.enumerate()
.map(|(i, v)| {
let ty = columns
.get(i)
.and_then(|c| c.type_text.as_deref())
.unwrap_or("");
coerce(ty, v)
})
.collect()
})
.collect();
// manifest 描述整个结果集,下一块的链接却在 result 里。后续 GET 返回的
// 是 ResultData 本身,没有 statement/status/manifest 外壳。
let manifest = resp.manifest.as_ref();
let truncated = manifest.and_then(|m| m.truncated).unwrap_or(false);
if truncated {
anyhow::bail!(
"Databricks returned a truncated result; row_limit must be raised or the query rewritten"
);
}
let mut next_link = resp
.result
.as_ref()
.and_then(|r| r.next_chunk_internal_link.clone());
let base = reqwest::Url::parse(&self.conn.base)?;
while let Some(link) = next_link.take() {
// internal_link 是相对于 workspace 的路径,不是可直接发给 reqwest 的
// URL。解析后还要留在同一 workspace,才可带着它的 bearer token 请求。
let url = base.join(&link)?;
anyhow::ensure!(
url.origin() == base.origin(),
"Databricks chunk link points outside the workspace"
);
let chunk: ResultData = client
.get(url)
.bearer_auth(&self.conn.token)
.send()
.await?
.error_for_status()?
.json()
.await?;
let chunk_rows = chunk
.data_array
.ok_or_else(|| anyhow::anyhow!("Databricks result chunk returned no data_array"))?;
rows.extend(chunk_rows.into_iter().map(|row| {
row.iter()
.enumerate()
.map(|(i, v)| {
let ty = columns
.get(i)
.and_then(|c| c.type_text.as_deref())
.unwrap_or("");
coerce(ty, v)
})
.collect()
}));
next_link = chunk.next_chunk_internal_link;
}
// schema 读按 docs 说不该见到 truncated=true;total_row_count 是每次都给的,
// 校验要放在所有 chunk 都收齐之后:第一次响应给的 total 是整个结果集的总数,
// 不是第一个 chunk 的总数
if let (Some(manifest), None) = (manifest, row_limit) {
let expected = manifest.total_row_count;
if let Some(expected) = expected {
let got: u64 = rows.len() as u64;
if got != expected {
anyhow::bail!(
"Databricks schema read returned {got} rows but manifest reports {expected}"
);
}
}
}
Ok((columns.into_iter().map(|c| c.name).collect(), rows))
}
}
/// 引擎说的是「这个 catalog / 表不存在(或看不见)」。认的是 Databricks 的错误类名,
/// 它们是协议的一部分(`[TABLE_OR_VIEW_NOT_FOUND] …`),不是措辞
fn is_not_found(e: &anyhow::Error) -> bool {
const CLASSES: [&str; 4] = [
"TABLE_OR_VIEW_NOT_FOUND",
"NO_SUCH_CATALOG_EXCEPTION",
"CATALOG_NOT_FOUND",
"SCHEMA_NOT_FOUND",
];
let text = e.to_string();
CLASSES.iter().any(|c| text.contains(c))
}
#[async_trait::async_trait]
impl QueryEngine for DatabricksEngine {
async fn test(&self) -> anyhow::Result<()> {
self.run("SELECT 1", Some(ROW_CAP + 1)).await.map(|_| ())
}
async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
// 两条候选,先准后全。真实集群上 `main`.information_schema.columns 回过
// TABLE_OR_VIEW_NOT_FOUND(#241),所以第一条**说找不到**时退到 system 那份。
//
// 只有「找不到」才退:令牌不对、仓库停着、超时,换一条语句也还是这样,再跑一遍
// 只是让点按钮的人多等半分钟。退过去读到 0 行也算失败——两份 information_schema
// 都只列令牌看得见的对象,system 那份还不含 hive_metastore,看不见的 catalog
// 在那里是「成功地没有行」;当成功的话,结构文档会被一份空的覆盖掉。
// 失败时每条试过的写法连同引擎的原话都带上
//
// row_limit 不传(#703):information_schema.columns 每行是一列,宽 catalog
// 上的 schema 文档会被悄悄截到 201 行;现在 `run` 自己跟 `next_chunk_internal_link`
// 一直拉到 `manifest.total_row_count` 行,多分块也收齐
let queries = self.schema_queries();
let mut tried: Vec<String> = Vec::new();
for (i, (label, sql)) in queries.iter().enumerate() {
match self.run(sql, None).await {
Ok((_, rows)) if rows.is_empty() && i > 0 => {
tried.push(format!(
"{label}: no columns for this catalog — the token cannot see it, \
or it is not managed by Unity Catalog"
));
break;
}
Ok((_, rows)) => {
return Ok(rows.into_iter().map(super::trino::schema_row).collect())
}
Err(e) => {
let missing = is_not_found(&e);
tried.push(format!("{label}: {e}"));
if !missing {
break;
}
}
}
}
anyhow::bail!("no readable information_schema ({})", tried.join("; "))
}
async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {
let (columns, rows) = self.run(&wrap_limit(sql), Some(ROW_CAP + 1)).await?;
let (rows, truncated) = truncate_rows(rows);
Ok(QueryResult {
rows: rows_to_json_lines(&columns, &rows),
truncated,
})
}
}
#[cfg(test)]
mod tests {
use super::super::conn::DatabricksConn;
use super::super::QueryEngine;
use super::{DatabricksEngine, ROW_CAP};
use serde_json::json;
use wiremock::matchers::{
body_partial_json, body_string_contains, header, method, path, query_param,
};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn conn(server: &MockServer) -> DatabricksConn {
DatabricksConn::parse(&format!(
"databricks://:dapi-test@{}/sql/1.0/warehouses/wh1?catalog=main&ssl=false",
server.uri().trim_start_matches("http://")
))
.unwrap()
}
#[tokio::test]
async fn polls_until_succeeded_and_restores_types() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(header("authorization", "Bearer dapi-test"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s1",
"status": { "state": "PENDING" }
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/2.0/sql/statements/s1"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s1",
"status": { "state": "SUCCEEDED" },
"manifest": { "schema": { "columns": [
{ "name": "region", "type_text": "STRING", "position": 0 },
{ "name": "total", "type_text": "DECIMAL(12,2)", "position": 1 },
{ "name": "active", "type_text": "BOOLEAN", "position": 2 }
] } },
"result": { "data_array": [ ["east", "12.50", "true"], ["west", null, "false"] ] }
})))
.expect(1)
.mount(&server)
.await;
let out = DatabricksEngine::new(conn(&server))
.execute("SELECT region, total, active FROM orders")
.await
.unwrap();
assert_eq!(
out.rows,
vec![
r#"{"region":"east","total":12.5,"active":true}"#,
r#"{"region":"west","total":null,"active":false}"#
]
);
}
#[tokio::test]
async fn a_failed_statement_reports_the_message() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s2",
"status": { "state": "FAILED", "error": { "error_code": "BAD_REQUEST", "message": "TABLE_OR_VIEW_NOT_FOUND: nope" } }
})))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT * FROM nope")
.await
.unwrap_err()
.to_string();
assert!(err.contains("TABLE_OR_VIEW_NOT_FOUND"), "{err}");
}
/// catalog 级那份读不到(真实集群回过 TABLE_OR_VIEW_NOT_FOUND)时退到 system 那份。
#[tokio::test]
async fn the_schema_read_falls_back_to_the_system_information_schema() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("`main`.information_schema.columns"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s3",
"status": { "state": "FAILED", "error": {
"error_code": "BAD_REQUEST",
"message": "[TABLE_OR_VIEW_NOT_FOUND] The table or view `main`.`information_schema`.`columns` cannot be found."
} }
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("system.information_schema.columns"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s4",
"status": { "state": "SUCCEEDED" },
"manifest": { "schema": { "columns": [
{ "name": "table_schema", "type_text": "STRING", "position": 0 },
{ "name": "table_name", "type_text": "STRING", "position": 1 },
{ "name": "column_name", "type_text": "STRING", "position": 2 },
{ "name": "data_type", "type_text": "STRING", "position": 3 },
{ "name": "comment", "type_text": "STRING", "position": 4 }
] } },
"result": { "data_array": [
["default", "orders", "amount", "DECIMAL(12,2)", "订单金额"]
] }
})))
.expect(1)
.mount(&server)
.await;
let cols = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap();
assert_eq!(cols.len(), 1);
assert_eq!(cols[0].schema, "default");
assert_eq!(cols[0].table, "orders");
assert_eq!(cols[0].column, "amount");
assert_eq!(cols[0].data_type, "DECIMAL(12,2)");
assert_eq!(cols[0].comment.as_deref(), Some("订单金额"));
}
fn failed(code: &str, message: &str) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_json(json!({
"status": { "state": "FAILED", "error": { "error_code": code, "message": message } }
}))
}
/// 两条都不通时,错误带上每一条试过的写法和它自己的原话。
#[tokio::test]
async fn two_dead_ends_say_which_ones_were_tried() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("`main`.information_schema.columns"))
.respond_with(failed(
"BAD_REQUEST",
"[TABLE_OR_VIEW_NOT_FOUND] first words",
))
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("system.information_schema.columns"))
.respond_with(failed("PERMISSION_DENIED", "second words"))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap_err()
.to_string();
assert!(err.contains("catalog information_schema"), "{err}");
assert!(err.contains("first words"), "{err}");
assert!(err.contains("system information_schema"), "{err}");
assert!(err.contains("second words"), "{err}");
}
/// 令牌不对、仓库停着这类错误不是「找不到」:换一条语句也一样,只发一次
#[tokio::test]
async fn an_error_that_is_not_a_missing_table_is_not_retried() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(failed("PERMISSION_DENIED", "token lacks USE CATALOG"))
.expect(1)
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap_err()
.to_string();
assert!(err.contains("token lacks USE CATALOG"), "{err}");
assert!(!err.contains("system information_schema"), "{err}");
}
/// 退到 system 那份读到 0 行:看不见这个 catalog,或它不归 Unity Catalog 管。
/// 这是失败,不是一份空的结构——否则刷新会把原来的结构文档覆盖成空的
#[tokio::test]
async fn a_fallback_that_reads_no_columns_is_a_failure() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("`main`.information_schema.columns"))
.respond_with(failed(
"BAD_REQUEST",
"[TABLE_OR_VIEW_NOT_FOUND] cannot be found",
))
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("system.information_schema.columns"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s5",
"status": { "state": "SUCCEEDED" },
"manifest": { "schema": { "columns": [
{ "name": "table_schema", "type_text": "STRING", "position": 0 }
] } },
"result": { "data_array": [] }
})))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap_err()
.to_string();
assert!(err.contains("no columns for this catalog"), "{err}");
assert!(err.contains("cannot be found"), "{err}");
}
/// 宽 catalog 上的 schema 读按 docs 走 INLINE 多分块:第一个 chunk 给
/// `next_chunk_internal_link`,按它 GET 到第二个 chunk,没有下一页就停。
/// 之前对所有语句都传 row_limit=201(#703),宽 catalog 上 information_schema.columns
/// 一行一列,每张表第一个 200 列之后的列就被丢掉了;现在 `fetch_schema` 不传
/// row_limit,分块之间按 link 拉齐
#[tokio::test]
async fn a_schema_read_follows_chunks_until_exhausted() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("`main`.information_schema.columns"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s_chunk1",
"status": { "state": "SUCCEEDED" },
"manifest": {
"schema": { "columns": [
{ "name": "table_schema", "type_text": "STRING", "position": 0 },
{ "name": "table_name", "type_text": "STRING", "position": 1 },
{ "name": "column_name", "type_text": "STRING", "position": 2 },
{ "name": "data_type", "type_text": "STRING", "position": 3 },
{ "name": "comment", "type_text": "STRING", "position": 4 }
] },
"total_row_count": 3,
"total_chunk_count": 3,
"truncated": false
},
"result": {
"chunk_index": 0, "row_offset": 0, "row_count": 1,
"next_chunk_index": 1,
"next_chunk_internal_link": "/api/2.0/sql/statements/s_chunk1/result/chunks/1?row_offset=1&cursor=a%2Fb%2Bc",
"data_array": [["default", "wide_orders", "id", "BIGINT", "主键"]]
}
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/2.0/sql/statements/s_chunk1/result/chunks/1"))
.and(query_param("row_offset", "1"))
.and(query_param("cursor", "a/b+c"))
.and(header("authorization", "Bearer dapi-test"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"chunk_index": 1, "row_offset": 1, "row_count": 1,
"next_chunk_index": 2,
"next_chunk_internal_link": "/api/2.0/sql/statements/s_chunk1/result/chunks/2?row_offset=2",
"data_array": [["default", "wide_orders", "region", "STRING", null]]
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/2.0/sql/statements/s_chunk1/result/chunks/2"))
.and(query_param("row_offset", "2"))
.and(header("authorization", "Bearer dapi-test"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"chunk_index": 2, "row_offset": 2, "row_count": 1,
"data_array": [["default", "wide_orders", "amount", "DECIMAL(12,2)", null]]
})))
.expect(1)
.mount(&server)
.await;
let cols = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap();
assert_eq!(cols.len(), 3, "want 3 columns across chunks, got {cols:?}");
assert_eq!(cols[0].column, "id");
assert_eq!(cols[1].column, "region");
assert_eq!(cols[2].column, "amount");
let requests = server.received_requests().await.unwrap();
assert!(requests[0]
.body_json::<serde_json::Value>()
.unwrap()
.get("row_limit")
.is_none());
assert_eq!(
requests[1].url.query(),
Some("row_offset=1&cursor=a%2Fb%2Bc")
);
}
#[tokio::test]
async fn a_chat_query_reads_later_chunks_before_applying_the_row_cap() {
for total in [1, 3, ROW_CAP + 1] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_partial_json(json!({"row_limit": ROW_CAP + 1})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s_chat",
"status": { "state": "SUCCEEDED" },
"manifest": {
"schema": { "columns": [{ "name": "n", "type_text": "BIGINT" }] },
"total_row_count": total,
"truncated": false
},
"result": {
"data_array": [["0"]],
"next_chunk_internal_link": "/api/2.0/sql/statements/s_chat/result/chunks/1?row_offset=1"
}
})))
.expect(1)
.mount(&server)
.await;
let remaining: Vec<_> = (1..total).map(|n| vec![n.to_string()]).collect();
Mock::given(method("GET"))
.and(path("/api/2.0/sql/statements/s_chat/result/chunks/1"))
.and(query_param("row_offset", "1"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"chunk_index": 1, "row_offset": 1, "row_count": total - 1,
"data_array": remaining
})))
.expect(1)
.mount(&server)
.await;
let out = DatabricksEngine::new(conn(&server))
.execute("SELECT n FROM numbers")
.await
.unwrap();
let expected: Vec<_> = (0..total.min(ROW_CAP))
.map(|n| format!(r#"{{"n":{n}}}"#))
.collect();
assert_eq!(out.rows, expected);
assert_eq!(out.truncated, total > ROW_CAP);
}
}
#[tokio::test]
async fn a_later_chunk_failure_does_not_return_a_partial_query() {
for (response, expected) in [
(ResponseTemplate::new(503), "503"),
(
ResponseTemplate::new(200).set_body_json(json!({})),
"data_array",
),
(
ResponseTemplate::new(200).set_body_json(json!({
"status": { "state": "SUCCEEDED" },
"result": { "data_array": [["second"]] }
})),
"data_array",
),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"status": { "state": "SUCCEEDED" },
"result": {
"data_array": [["first"]],
"next_chunk_internal_link": "/api/2.0/sql/statements/s_failed/result/chunks/1"
}
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/2.0/sql/statements/s_failed/result/chunks/1"))
.respond_with(response)
.expect(1)
.mount(&server)
.await;
let error = DatabricksEngine::new(conn(&server))
.execute("SELECT n FROM numbers")
.await
.unwrap_err();
assert!(error.to_string().contains(expected), "{error}");
}
}
#[tokio::test]
async fn an_internal_chunk_link_cannot_send_the_token_to_another_origin() {
let foreign = MockServer::start().await;
for link in [
format!("{}/result/chunks/1", foreign.uri()),
format!("//{}/result/chunks/1", foreign.address()),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"status": { "state": "SUCCEEDED" },
"result": {
"data_array": [["first"]],
"next_chunk_internal_link": link
}
})))
.expect(1)
.mount(&server)
.await;
let error = DatabricksEngine::new(conn(&server))
.execute("SELECT n FROM numbers")
.await
.unwrap_err();
assert!(
error.to_string().contains("outside the workspace"),
"{error}"
);
}
assert!(foreign.received_requests().await.unwrap().is_empty());
}
/// 跑 chat 查询时(带 row_limit),服务端真的把行砍了:manifest.truncated=true
/// 应当直接失败,而不是带着 < ROW_CAP 行假装「查询成功」
#[tokio::test]
async fn a_chat_query_bails_loudly_when_the_service_says_truncated() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s_trunc",
"status": { "state": "SUCCEEDED" },
"manifest": {
"schema": { "columns": [
{ "name": "n", "type_text": "BIGINT", "position": 0 }
] },
"total_row_count": 1000,
"truncated": true
},
"result": { "data_array": [ ["1"], ["2"] ] }
})))
.expect(1)
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT n FROM big")
.await
.unwrap_err()
.to_string();
assert!(err.contains("truncated"), "{err}");
assert!(err.contains("row_limit"), "{err}");
}
/// schema 读 manifest 报的 total_row_count 跟实际收到的行数对不上——服务端
/// 偷偷改 protocol,或中途丢包:fail-fast,免得把残缺的 schema 灌进结构文档
#[tokio::test]
async fn a_schema_read_fails_when_total_row_count_does_not_match() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.and(body_string_contains("`main`.information_schema.columns"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s_count_mismatch",
"status": { "state": "SUCCEEDED" },
"manifest": {
"schema": { "columns": [
{ "name": "table_schema", "type_text": "STRING", "position": 0 },
{ "name": "table_name", "type_text": "STRING", "position": 1 },
{ "name": "column_name", "type_text": "STRING", "position": 2 },
{ "name": "data_type", "type_text": "STRING", "position": 3 },
{ "name": "comment", "type_text": "STRING", "position": 4 }
] },
"total_row_count": 99
},
"result": { "data_array": [
["default", "orders", "id", "BIGINT", null]
] }
})))
.expect(1)
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap_err()
.to_string();
assert!(err.contains("schema read returned 1 rows"), "{err}");
assert!(err.contains("99"), "{err}");
}
/// 401/403 一类:token 失效或 PAT 撤销。`error_for_status()` 会把非 2xx
/// 抛成 reqwest 错误——表现上等价「连接失败 / 鉴权失败」。把这层向上层送
/// 调用者时仍可分辨:「reqwest」还是业务 message 的边界
#[tokio::test]
async fn an_unauthorized_http_status_is_surfaced_as_a_connection_error() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(401).set_body_string("Unauthorized"))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
// reqwest 的 error_for_status 把 status 包成 builder 错误;我们要确保
// 调用者看到的是 HTTP 401 字样,不是「Snowflake: …」的错形态
assert!(err.contains("401"), "{err}");
}
/// 5xx 一类(warehouse suspended / cluster down)走同一条 error_for_status
/// 路径——表面上仍是 reqwest 错误,需要可识别的 status code
#[tokio::test]
async fn a_5xx_response_is_surfaced_with_status_code() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(
ResponseTemplate::new(503)
.set_body_string("Service Unavailable — warehouse suspended"),
)
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(err.contains("503"), "{err}");
}
/// 用户主动取消(cancel button 或 stmt cancellation)是 Databricks 的合法
/// 状态:`CANCELED`。业务错误带 message 后应能让上层识别这是「取消」
/// 而不是「失败」,所以 message 应完整透传
#[tokio::test]
async fn a_canceled_statement_carries_a_distinct_message() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s3",
"status": { "state": "CANCELED", "error": { "error_code": "CANCELED", "message": "Statement was canceled by user." } }
})))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(err.contains("CANCELED"), "{err}");
assert!(err.contains("canceled by user"), "{err}");
}
/// PENDING 响应里若缺失 `statement_id`,现有逻辑会以「Databricks returned
/// no statement_id」抛出——把这条兜底钉住,免得一改 polling 就把它吞掉
#[tokio::test]
async fn pending_state_without_statement_id_is_surfaced() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"status": { "state": "PENDING" }
})))
.mount(&server)
.await;
let err = DatabricksEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(
err.contains("no statement_id"),
"PENDING 但没 statement_id 应被显式提示,避免 polling 失效时不报错:{err}"
);
}
/// `SUCCEEDED` 但 `result.data_array` 是空(0 行查询 / DDL 走过 execute 路径),
/// rows 取 []、columns 也取空向量——`rows_to_json_lines` 仍要返回空集、不崩
#[tokio::test]
async fn a_succeeded_statement_with_no_rows_returns_empty() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/2.0/sql/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"statement_id": "s4",
"status": { "state": "SUCCEEDED" },
"manifest": { "schema": { "columns": [] } },
"result": { "data_array": [] }
})))
.mount(&server)
.await;
let out = DatabricksEngine::new(conn(&server))
.execute("SELECT 1 WHERE 1=0")
.await
.unwrap();
assert!(out.rows.is_empty(), "{out:?}");
assert!(!out.truncated);
}
}