World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! Snowflake SQL API v2(`/api/v2/statements`)。同步提交(`async=false`)拿不完的
//! 语句回 202,沿 statementHandle 轮询。值全是字符串,按 rowType 还原数与布尔。
//!
//! 只收令牌,不收密码:programmatic access token 或 OAuth。密钥对 JWT 要本地签名,
//! 这一版不做——见 `conn.rs`。
use super::conn::SnowflakeConn;
use super::{
coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult,
SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS,
};
use reqwest::StatusCode;
use serde::Deserialize;
use serde_json::json;
use std::time::{Duration, Instant};
pub struct SnowflakeEngine {
conn: SnowflakeConn,
}
#[derive(Deserialize)]
struct StatementResponse {
#[serde(rename = "resultSetMetaData")]
meta: Option<Meta>,
data: Option<Vec<Vec<serde_json::Value>>>,
message: Option<String>,
code: Option<String>,
#[serde(rename = "statementHandle")]
handle: Option<String>,
}
#[derive(Deserialize, Clone)]
struct Meta {
#[serde(rename = "rowType")]
row_type: Vec<RowType>,
#[serde(rename = "numRows")]
num_rows: Option<u64>,
/// 每条只描述分区的行数和大小;分区号是数组下标,后续请求使用 statementHandle。
#[serde(rename = "partitionInfo")]
partition_info: Option<Vec<PartitionInfo>>,
}
#[derive(Deserialize, Clone)]
struct RowType {
name: String,
#[serde(rename = "type")]
ty: String,
}
/// 每个分区的行数,用于核对按数组下标获取的完整结果集。
#[derive(Deserialize, Clone)]
struct PartitionInfo {
#[serde(rename = "rowCount")]
row_count: u64,
}
impl SnowflakeEngine {
pub fn new(conn: SnowflakeConn) -> Self {
Self { conn }
}
fn request(&self, r: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
r.bearer_auth(&self.conn.token)
.header("X-Snowflake-Authorization-Token-Type", self.conn.token_type)
.header("Accept", "application/json")
}
async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
let client = super::http()?;
let mut body = json!({
"statement": sql,
"timeout": STATEMENT_TIMEOUT_SECS,
"parameters": { "MULTI_STATEMENT_COUNT": "1" },
});
for (key, value) in [
("database", &self.conn.database),
("schema", &self.conn.schema),
("warehouse", &self.conn.warehouse),
("role", &self.conn.role),
] {
if let Some(v) = value {
body[key] = json!(v);
}
}
let mut http = self
.request(client.post(format!("{}/api/v2/statements?async=false", self.conn.base)))
.json(&body)
.send()
.await?;
let started = Instant::now();
// 202 = 还在跑;其余非 2xx 的 body 里带 message
while http.status() == StatusCode::ACCEPTED {
let partial: StatementResponse = http.json().await?;
let handle = partial.handle.ok_or_else(|| {
anyhow::anyhow!("Snowflake returned 202 without a statementHandle")
})?;
if started.elapsed() > HTTP_POLL_BUDGET {
anyhow::bail!(
"Snowflake statement did not finish within {}s",
HTTP_POLL_BUDGET.as_secs()
);
}
tokio::time::sleep(Duration::from_secs(1)).await;
http = self
.request(client.get(format!("{}/api/v2/statements/{handle}", self.conn.base)))
.send()
.await?;
}
if !http.status().is_success() {
let status = http.status();
let text = http.text().await.unwrap_or_default();
let msg = serde_json::from_str::<StatementResponse>(&text)
.ok()
.and_then(|r| r.message)
.unwrap_or(text);
anyhow::bail!("Snowflake {status}: {msg}");
}
let resp: StatementResponse = http.json().await?;
if let (Some(code), Some(message)) = (&resp.code, &resp.message) {
// 2xx 里也可能带业务错误码;090001 是 "statement executed successfully"
if code != "090001" && resp.meta.is_none() {
anyhow::bail!("Snowflake {code}: {message}");
}
}
// 把 meta 拷出一份再消费;下面既要从 partition_info 拉分块,
// 又要从 row_type 还原类型,两个都要用
let meta = resp.meta.clone();
let types: Vec<RowType> = meta
.as_ref()
.map(|m| m.row_type.clone())
.unwrap_or_default();
let partitions: Vec<PartitionInfo> = meta
.as_ref()
.and_then(|m| m.partition_info.clone())
.unwrap_or_default();
let expected_total_rows: Option<u64> = meta.as_ref().and_then(|m| m.num_rows);
let mut rows: Vec<Vec<serde_json::Value>> = resp
.data
.unwrap_or_default()
.into_iter()
.map(|row| {
row.iter()
.enumerate()
.map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))
.collect()
})
.collect();
// partitionInfo 没有 URL。首个分区已在响应中,后续按 statementHandle 和
// 从 1 开始的 partition 参数读取;后续响应不再带列元数据,沿用首块的类型。
let mut seen_partition_rows: u64 = rows.len() as u64;
if partitions.len() > 1 {
let handle = resp.handle.as_deref().ok_or_else(|| {
anyhow::anyhow!("Snowflake returned multiple partitions without a statementHandle")
})?;
for partition in 1..partitions.len() {
let chunk_resp: StatementResponse = self
.request(client.get(format!(
"{}/api/v2/statements/{handle}?partition={partition}",
self.conn.base
)))
.send()
.await?
.error_for_status()?
.json()
.await?;
let chunk_rows = chunk_resp.data.unwrap_or_default();
seen_partition_rows = seen_partition_rows.saturating_add(chunk_rows.len() as u64);
rows.extend(chunk_rows.into_iter().map(|row| {
row.iter()
.enumerate()
.map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))
.collect()
}));
}
}
// numRows 是整个结果集的总数(不是当前 chunk 的);分区 rowCount 之和也应该是它
// ——任一对不上都说明服务端偷偷改 protocol 或中途丢包,残缺数据不要灌进结构文档
if let Some(expected) = expected_total_rows {
if seen_partition_rows != expected {
anyhow::bail!(
"Snowflake schema read returned {seen_partition_rows} rows across partitions but numRows reports {expected}"
);
}
}
let partition_total: u64 = partitions.iter().map(|p| p.row_count).sum();
if !partitions.is_empty() && partition_total != seen_partition_rows {
anyhow::bail!(
"Snowflake partition row counts sum to {partition_total} but received {seen_partition_rows} rows"
);
}
Ok((types.into_iter().map(|t| t.name).collect(), rows))
}
}
#[async_trait::async_trait]
impl QueryEngine for SnowflakeEngine {
async fn test(&self) -> anyhow::Result<()> {
self.run("SELECT 1").await.map(|_| ())
}
async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
let database = self.conn.database.as_deref().ok_or_else(|| {
anyhow::anyhow!("snowflake://: put the database in the connection string (snowflake://:TOKEN@account/DATABASE) so the schema can be read")
})?;
let schema_filter = self
.conn
.schema
.as_deref()
.map(|s| format!(" AND table_schema = {}", sql_literal(s)))
.unwrap_or_default();
let sql = format!(
"SELECT table_schema, table_name, column_name, data_type, comment \
FROM \"{}\".information_schema.columns \
WHERE table_schema <> 'INFORMATION_SCHEMA'{schema_filter} \
ORDER BY table_schema, table_name, ordinal_position",
database.replace('"', "\"\"")
);
let (_, rows) = self.run(&sql).await?;
Ok(rows.into_iter().map(super::trino::schema_row).collect())
}
async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {
let (columns, rows) = self.run(&wrap_limit(sql)).await?;
let (rows, truncated) = truncate_rows(rows);
Ok(QueryResult {
rows: rows_to_json_lines(&columns, &rows),
truncated,
})
}
}
#[cfg(test)]
mod tests {
use super::super::conn::SnowflakeConn;
use super::super::QueryEngine;
use super::SnowflakeEngine;
use serde_json::json;
use std::io::Write;
use wiremock::matchers::{header, method, path, query_param};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn conn(server: &MockServer) -> SnowflakeConn {
SnowflakeConn::parse(&format!(
"snowflake://:pat-test@{}/ANALYTICS/PUBLIC?warehouse=WH&ssl=false",
server.uri().trim_start_matches("http://")
))
.unwrap()
}
// The SQL API compresses later partitions and omits their metadata.
fn compressed_partition(data: serde_json::Value) -> ResponseTemplate {
let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
gzip.write_all(
serde_json::to_string(&json!({ "data": data }))
.unwrap()
.as_bytes(),
)
.unwrap();
ResponseTemplate::new(200)
.insert_header("Content-Encoding", "gzip")
.set_body_bytes(gzip.finish().unwrap())
}
#[tokio::test]
async fn a_synchronous_answer_is_typed_by_row_type() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.and(header("authorization", "Bearer pat-test"))
.and(header(
"X-Snowflake-Authorization-Token-Type",
"PROGRAMMATIC_ACCESS_TOKEN",
))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"resultSetMetaData": { "numRows": 1, "rowType": [
{ "name": "REGION", "type": "text" },
{ "name": "TOTAL", "type": "fixed", "scale": 2 }
] },
"data": [ ["east", "42.10"] ],
"code": "090001",
"statementHandle": "h1",
"message": "Statement executed successfully."
})))
.expect(1)
.mount(&server)
.await;
let out = SnowflakeEngine::new(conn(&server))
.execute("SELECT region, total FROM orders")
.await
.unwrap();
assert_eq!(out.rows, vec![r#"{"REGION":"east","TOTAL":42.1}"#]);
}
#[tokio::test]
async fn a_202_is_polled_and_its_partitions_keep_the_first_rows_types() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(202).set_body_json(json!({
"code": "333334", "statementHandle": "h2", "message": "Asynchronous execution in progress."
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v2/statements/h2"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"resultSetMetaData": {
"numRows": 3,
"rowType": [ { "name": "N", "type": "fixed" } ],
"partitionInfo": [ { "rowCount": 1 }, { "rowCount": 2 } ]
},
"data": [ ["1"] ], "code": "090001", "statementHandle": "h2"
})))
.up_to_n_times(1)
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v2/statements/h2"))
.and(query_param("partition", "1"))
.and(header("authorization", "Bearer pat-test"))
.and(header(
"X-Snowflake-Authorization-Token-Type",
"PROGRAMMATIC_ACCESS_TOKEN",
))
.respond_with(compressed_partition(json!([["2"], [null]])))
.expect(1)
.mount(&server)
.await;
let out = SnowflakeEngine::new(conn(&server))
.execute("SELECT 1 AS n")
.await
.unwrap();
assert_eq!(out.rows, vec![r#"{"N":1}"#, r#"{"N":2}"#, r#"{"N":null}"#]);
assert!(!out.truncated);
}
#[tokio::test]
async fn an_error_body_is_surfaced() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(422).set_body_json(json!({
"code": "002003", "message": "SQL compilation error: Object 'NOPE' does not exist"
})))
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT * FROM nope")
.await
.unwrap_err()
.to_string();
assert!(err.contains("does not exist"), "{err}");
}
/// partitionInfo only describes row counts and sizes. Partition zero is
/// inline; subsequent partitions use the statement handle and their index.
#[tokio::test]
async fn a_schema_read_follows_partitions_until_exhausted() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"resultSetMetaData": {
"numRows": 3,
"rowType": [
{ "name": "TABLE_SCHEMA", "type": "text" },
{ "name": "TABLE_NAME", "type": "text" },
{ "name": "COLUMN_NAME", "type": "text" },
{ "name": "DATA_TYPE", "type": "text" },
{ "name": "COMMENT", "type": "text" }
],
"partitionInfo": [
{ "rowCount": 1, "uncompressedSize": 120 },
{ "rowCount": 1, "uncompressedSize": 130, "compressedSize": 80 },
{ "rowCount": 1, "uncompressedSize": 140, "compressedSize": 90 }
]
},
"data": [
["default", "wide_orders", "id", "BIGINT", "主键"]
],
"code": "090001",
"statementHandle": "h3",
"message": "Statement executed successfully."
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v2/statements/h3"))
.and(query_param("partition", "1"))
.and(header("authorization", "Bearer pat-test"))
.respond_with(compressed_partition(json!([[
"default",
"wide_orders",
"region",
"STRING",
null
]])))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v2/statements/h3"))
.and(query_param("partition", "2"))
.and(header("authorization", "Bearer pat-test"))
.respond_with(compressed_partition(json!([[
"default",
"wide_orders",
"amount",
"DECIMAL(12,2)",
null
]])))
.expect(1)
.mount(&server)
.await;
let cols = SnowflakeEngine::new(conn(&server))
.fetch_schema()
.await
.unwrap();
assert_eq!(
cols.len(),
3,
"want 3 columns across partitions, got {cols:?}"
);
assert_eq!(cols[0].column, "id");
assert_eq!(cols[1].column, "region");
assert_eq!(cols[2].column, "amount");
}
#[tokio::test]
async fn multiple_partitions_require_a_statement_handle() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"resultSetMetaData": {
"rowType": [{ "name": "N", "type": "fixed" }],
"partitionInfo": [{ "rowCount": 1 }, { "rowCount": 1 }]
},
"data": [["1"]]
})))
.expect(1)
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT n FROM numbers")
.await
.unwrap_err()
.to_string();
assert!(
err.contains("multiple partitions without a statementHandle"),
"{err}"
);
}
#[tokio::test]
async fn failed_or_incomplete_partitions_do_not_return_partial_rows() {
for (response, expected_error) in [
(ResponseTemplate::new(503), "503"),
(compressed_partition(json!([])), "numRows reports 2"),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"resultSetMetaData": {
"numRows": 2,
"rowType": [{ "name": "N", "type": "fixed" }],
"partitionInfo": [{ "rowCount": 1 }, { "rowCount": 1 }]
},
"data": [["1"]], "statementHandle": "h4"
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/api/v2/statements/h4"))
.and(query_param("partition", "1"))
.respond_with(response)
.expect(1)
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT n FROM numbers")
.await
.unwrap_err()
.to_string();
assert!(err.contains(expected_error), "{err}");
}
}
/// 401/403 一类(auth、权限)也是 status 路径——message 字段照样送出。
/// 与 422 的区别仅在状态码,调用者拿到的错误形态应保持一致(都是
/// `Snowflake {status}: {message}`),便于上层按错误形态分类
#[tokio::test]
async fn auth_failure_body_is_surfaced() {
for status in [401u16, 403, 503] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(status).set_body_json(json!({
"code": "390100",
"message": format!("HTTP {status}: not authorized — token expired or missing role"),
})))
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(err.contains(&status.to_string()), "{err}");
assert!(err.contains("token expired"), "{err}");
}
}
/// 5xx 一类(如 Snowflake 内部 504 / 503 + GW 错配)时 body 不是 JSON——
/// 现有逻辑会尝试 JSON.parse 然后退回原文,所以「消息文案」仍能送达
/// 调用者。把这层兜底路径钉住,免得被人「美化」掉
#[tokio::test]
async fn non_json_error_body_falls_back_to_plain_text() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(
ResponseTemplate::new(500).set_body_string("Internal error: warehouse suspended\n"),
)
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(err.contains("500"), "{err}");
assert!(
err.contains("warehouse suspended"),
"非 JSON body 应原样透传,给上层一份能落库的描述:{err}"
);
}
/// 2xx body 里若带非「成功」code(如 query canceled 取消信号),必须以业务
/// 错误形态抛出,不让屏蔽在「看似 OK」的响应里——这是 Snowflake 的双层错误
/// 模型,应当与 4xx/5xx 共用同一条 message 通道
#[tokio::test]
async fn business_error_code_in_a_2xx_body_is_bubbled_up() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v2/statements"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"code": "000605",
"message": "Statement was canceled by user.",
// 没有 resultSetMetaData,与状态一致——成功响应必有 meta 或 data
})))
.mount(&server)
.await;
let err = SnowflakeEngine::new(conn(&server))
.execute("SELECT 1")
.await
.unwrap_err()
.to_string();
assert!(err.contains("000605"), "{err}");
assert!(err.contains("canceled"), "{err}");
}
}