Skip to content

World's first open-source enterprise world model.

22.1 KB
Raw Blame
//! Snowflake SQL API v2(`/api/v2/statements`)。同步提交(`async=false`)拿不完的
//! 语句回 202,沿 statementHandle 轮询。值全是字符串,按 rowType 还原数与布尔。
//!
//! 只收令牌,不收密码:programmatic access token 或 OAuth。密钥对 JWT 要本地签名,
//! 这一版不做——见 `conn.rs`。

use super::conn::SnowflakeConn;
use super::{
    coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult,
    SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS,
};
use reqwest::StatusCode;
use serde::Deserialize;
use serde_json::json;
use std::time::{Duration, Instant};

pub struct SnowflakeEngine {
    conn: SnowflakeConn,
}

#[derive(Deserialize)]
struct StatementResponse {
    #[serde(rename = "resultSetMetaData")]
    meta: Option<Meta>,
    data: Option<Vec<Vec<serde_json::Value>>>,
    message: Option<String>,
    code: Option<String>,
    #[serde(rename = "statementHandle")]
    handle: Option<String>,
}

#[derive(Deserialize, Clone)]
struct Meta {
    #[serde(rename = "rowType")]
    row_type: Vec<RowType>,
    #[serde(rename = "numRows")]
    num_rows: Option<u64>,
    /// 每条只描述分区的行数和大小;分区号是数组下标,后续请求使用 statementHandle。
    #[serde(rename = "partitionInfo")]
    partition_info: Option<Vec<PartitionInfo>>,
}

#[derive(Deserialize, Clone)]
struct RowType {
    name: String,
    #[serde(rename = "type")]
    ty: String,
}

/// 每个分区的行数,用于核对按数组下标获取的完整结果集。
#[derive(Deserialize, Clone)]
struct PartitionInfo {
    #[serde(rename = "rowCount")]
    row_count: u64,
}

impl SnowflakeEngine {
    pub fn new(conn: SnowflakeConn) -> Self {
        Self { conn }
    }

    fn request(&self, r: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
        r.bearer_auth(&self.conn.token)
            .header("X-Snowflake-Authorization-Token-Type", self.conn.token_type)
            .header("Accept", "application/json")
    }

    async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
        let client = super::http()?;
        let mut body = json!({
            "statement": sql,
            "timeout": STATEMENT_TIMEOUT_SECS,
            "parameters": { "MULTI_STATEMENT_COUNT": "1" },
        });
        for (key, value) in [
            ("database", &self.conn.database),
            ("schema", &self.conn.schema),
            ("warehouse", &self.conn.warehouse),
            ("role", &self.conn.role),
        ] {
            if let Some(v) = value {
                body[key] = json!(v);
            }
        }
        let mut http = self
            .request(client.post(format!("{}/api/v2/statements?async=false", self.conn.base)))
            .json(&body)
            .send()
            .await?;
        let started = Instant::now();
        // 202 = 还在跑;其余非 2xx 的 body 里带 message
        while http.status() == StatusCode::ACCEPTED {
            let partial: StatementResponse = http.json().await?;
            let handle = partial.handle.ok_or_else(|| {
                anyhow::anyhow!("Snowflake returned 202 without a statementHandle")
            })?;
            if started.elapsed() > HTTP_POLL_BUDGET {
                anyhow::bail!(
                    "Snowflake statement did not finish within {}s",
                    HTTP_POLL_BUDGET.as_secs()
                );
            }
            tokio::time::sleep(Duration::from_secs(1)).await;
            http = self
                .request(client.get(format!("{}/api/v2/statements/{handle}", self.conn.base)))
                .send()
                .await?;
        }
        if !http.status().is_success() {
            let status = http.status();
            let text = http.text().await.unwrap_or_default();
            let msg = serde_json::from_str::<StatementResponse>(&text)
                .ok()
                .and_then(|r| r.message)
                .unwrap_or(text);
            anyhow::bail!("Snowflake {status}: {msg}");
        }
        let resp: StatementResponse = http.json().await?;
        if let (Some(code), Some(message)) = (&resp.code, &resp.message) {
            // 2xx 里也可能带业务错误码;090001 是 "statement executed successfully"
            if code != "090001" && resp.meta.is_none() {
                anyhow::bail!("Snowflake {code}: {message}");
            }
        }
        // 把 meta 拷出一份再消费;下面既要从 partition_info 拉分块,
        // 又要从 row_type 还原类型,两个都要用
        let meta = resp.meta.clone();
        let types: Vec<RowType> = meta
            .as_ref()
            .map(|m| m.row_type.clone())
            .unwrap_or_default();
        let partitions: Vec<PartitionInfo> = meta
            .as_ref()
            .and_then(|m| m.partition_info.clone())
            .unwrap_or_default();
        let expected_total_rows: Option<u64> = meta.as_ref().and_then(|m| m.num_rows);
        let mut rows: Vec<Vec<serde_json::Value>> = resp
            .data
            .unwrap_or_default()
            .into_iter()
            .map(|row| {
                row.iter()
                    .enumerate()
                    .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))
                    .collect()
            })
            .collect();
        // partitionInfo 没有 URL。首个分区已在响应中,后续按 statementHandle 和
        // 从 1 开始的 partition 参数读取;后续响应不再带列元数据,沿用首块的类型。
        let mut seen_partition_rows: u64 = rows.len() as u64;
        if partitions.len() > 1 {
            let handle = resp.handle.as_deref().ok_or_else(|| {
                anyhow::anyhow!("Snowflake returned multiple partitions without a statementHandle")
            })?;
            for partition in 1..partitions.len() {
                let chunk_resp: StatementResponse = self
                    .request(client.get(format!(
                        "{}/api/v2/statements/{handle}?partition={partition}",
                        self.conn.base
                    )))
                    .send()
                    .await?
                    .error_for_status()?
                    .json()
                    .await?;
                let chunk_rows = chunk_resp.data.unwrap_or_default();
                seen_partition_rows = seen_partition_rows.saturating_add(chunk_rows.len() as u64);
                rows.extend(chunk_rows.into_iter().map(|row| {
                    row.iter()
                        .enumerate()
                        .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))
                        .collect()
                }));
            }
        }
        // numRows 是整个结果集的总数(不是当前 chunk 的);分区 rowCount 之和也应该是它
        // ——任一对不上都说明服务端偷偷改 protocol 或中途丢包,残缺数据不要灌进结构文档
        if let Some(expected) = expected_total_rows {
            if seen_partition_rows != expected {
                anyhow::bail!(
                    "Snowflake schema read returned {seen_partition_rows} rows across partitions but numRows reports {expected}"
                );
            }
        }
        let partition_total: u64 = partitions.iter().map(|p| p.row_count).sum();
        if !partitions.is_empty() && partition_total != seen_partition_rows {
            anyhow::bail!(
                "Snowflake partition row counts sum to {partition_total} but received {seen_partition_rows} rows"
            );
        }
        Ok((types.into_iter().map(|t| t.name).collect(), rows))
    }
}

#[async_trait::async_trait]
impl QueryEngine for SnowflakeEngine {
    async fn test(&self) -> anyhow::Result<()> {
        self.run("SELECT 1").await.map(|_| ())
    }

    async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
        let database = self.conn.database.as_deref().ok_or_else(|| {
            anyhow::anyhow!("snowflake://: put the database in the connection string (snowflake://:TOKEN@account/DATABASE) so the schema can be read")
        })?;
        let schema_filter = self
            .conn
            .schema
            .as_deref()
            .map(|s| format!(" AND table_schema = {}", sql_literal(s)))
            .unwrap_or_default();
        let sql = format!(
            "SELECT table_schema, table_name, column_name, data_type, comment \
             FROM \"{}\".information_schema.columns \
             WHERE table_schema <> 'INFORMATION_SCHEMA'{schema_filter} \
             ORDER BY table_schema, table_name, ordinal_position",
            database.replace('"', "\"\"")
        );
        let (_, rows) = self.run(&sql).await?;
        Ok(rows.into_iter().map(super::trino::schema_row).collect())
    }

    async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {
        let (columns, rows) = self.run(&wrap_limit(sql)).await?;
        let (rows, truncated) = truncate_rows(rows);
        Ok(QueryResult {
            rows: rows_to_json_lines(&columns, &rows),
            truncated,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::super::conn::SnowflakeConn;
    use super::super::QueryEngine;
    use super::SnowflakeEngine;
    use serde_json::json;
    use std::io::Write;
    use wiremock::matchers::{header, method, path, query_param};
    use wiremock::{Mock, MockServer, ResponseTemplate};

    fn conn(server: &MockServer) -> SnowflakeConn {
        SnowflakeConn::parse(&format!(
            "snowflake://:pat-test@{}/ANALYTICS/PUBLIC?warehouse=WH&ssl=false",
            server.uri().trim_start_matches("http://")
        ))
        .unwrap()
    }

    // The SQL API compresses later partitions and omits their metadata.
    fn compressed_partition(data: serde_json::Value) -> ResponseTemplate {
        let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
        gzip.write_all(
            serde_json::to_string(&json!({ "data": data }))
                .unwrap()
                .as_bytes(),
        )
        .unwrap();
        ResponseTemplate::new(200)
            .insert_header("Content-Encoding", "gzip")
            .set_body_bytes(gzip.finish().unwrap())
    }

    #[tokio::test]
    async fn a_synchronous_answer_is_typed_by_row_type() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .and(header("authorization", "Bearer pat-test"))
            .and(header(
                "X-Snowflake-Authorization-Token-Type",
                "PROGRAMMATIC_ACCESS_TOKEN",
            ))
            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                "resultSetMetaData": { "numRows": 1, "rowType": [
                    { "name": "REGION", "type": "text" },
                    { "name": "TOTAL", "type": "fixed", "scale": 2 }
                ] },
                "data": [ ["east", "42.10"] ],
                "code": "090001",
                "statementHandle": "h1",
                "message": "Statement executed successfully."
            })))
            .expect(1)
            .mount(&server)
            .await;
        let out = SnowflakeEngine::new(conn(&server))
            .execute("SELECT region, total FROM orders")
            .await
            .unwrap();
        assert_eq!(out.rows, vec![r#"{"REGION":"east","TOTAL":42.1}"#]);
    }

    #[tokio::test]
    async fn a_202_is_polled_and_its_partitions_keep_the_first_rows_types() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(ResponseTemplate::new(202).set_body_json(json!({
                "code": "333334", "statementHandle": "h2", "message": "Asynchronous execution in progress."
            })))
            .expect(1)
            .mount(&server)
            .await;
        Mock::given(method("GET"))
            .and(path("/api/v2/statements/h2"))
            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                "resultSetMetaData": {
                    "numRows": 3,
                    "rowType": [ { "name": "N", "type": "fixed" } ],
                    "partitionInfo": [ { "rowCount": 1 }, { "rowCount": 2 } ]
                },
                "data": [ ["1"] ], "code": "090001", "statementHandle": "h2"
            })))
            .up_to_n_times(1)
            .expect(1)
            .mount(&server)
            .await;
        Mock::given(method("GET"))
            .and(path("/api/v2/statements/h2"))
            .and(query_param("partition", "1"))
            .and(header("authorization", "Bearer pat-test"))
            .and(header(
                "X-Snowflake-Authorization-Token-Type",
                "PROGRAMMATIC_ACCESS_TOKEN",
            ))
            .respond_with(compressed_partition(json!([["2"], [null]])))
            .expect(1)
            .mount(&server)
            .await;
        let out = SnowflakeEngine::new(conn(&server))
            .execute("SELECT 1 AS n")
            .await
            .unwrap();
        assert_eq!(out.rows, vec![r#"{"N":1}"#, r#"{"N":2}"#, r#"{"N":null}"#]);
        assert!(!out.truncated);
    }

    #[tokio::test]
    async fn an_error_body_is_surfaced() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(ResponseTemplate::new(422).set_body_json(json!({
                "code": "002003", "message": "SQL compilation error: Object 'NOPE' does not exist"
            })))
            .mount(&server)
            .await;
        let err = SnowflakeEngine::new(conn(&server))
            .execute("SELECT * FROM nope")
            .await
            .unwrap_err()
            .to_string();
        assert!(err.contains("does not exist"), "{err}");
    }

    /// partitionInfo only describes row counts and sizes. Partition zero is
    /// inline; subsequent partitions use the statement handle and their index.
    #[tokio::test]
    async fn a_schema_read_follows_partitions_until_exhausted() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                "resultSetMetaData": {
                    "numRows": 3,
                    "rowType": [
                        { "name": "TABLE_SCHEMA", "type": "text" },
                        { "name": "TABLE_NAME", "type": "text" },
                        { "name": "COLUMN_NAME", "type": "text" },
                        { "name": "DATA_TYPE", "type": "text" },
                        { "name": "COMMENT", "type": "text" }
                    ],
                    "partitionInfo": [
                        { "rowCount": 1, "uncompressedSize": 120 },
                        { "rowCount": 1, "uncompressedSize": 130, "compressedSize": 80 },
                        { "rowCount": 1, "uncompressedSize": 140, "compressedSize": 90 }
                    ]
                },
                "data": [
                    ["default", "wide_orders", "id", "BIGINT", "主键"]
                ],
                "code": "090001",
                "statementHandle": "h3",
                "message": "Statement executed successfully."
            })))
            .expect(1)
            .mount(&server)
            .await;
        Mock::given(method("GET"))
            .and(path("/api/v2/statements/h3"))
            .and(query_param("partition", "1"))
            .and(header("authorization", "Bearer pat-test"))
            .respond_with(compressed_partition(json!([[
                "default",
                "wide_orders",
                "region",
                "STRING",
                null
            ]])))
            .expect(1)
            .mount(&server)
            .await;
        Mock::given(method("GET"))
            .and(path("/api/v2/statements/h3"))
            .and(query_param("partition", "2"))
            .and(header("authorization", "Bearer pat-test"))
            .respond_with(compressed_partition(json!([[
                "default",
                "wide_orders",
                "amount",
                "DECIMAL(12,2)",
                null
            ]])))
            .expect(1)
            .mount(&server)
            .await;

        let cols = SnowflakeEngine::new(conn(&server))
            .fetch_schema()
            .await
            .unwrap();
        assert_eq!(
            cols.len(),
            3,
            "want 3 columns across partitions, got {cols:?}"
        );
        assert_eq!(cols[0].column, "id");
        assert_eq!(cols[1].column, "region");
        assert_eq!(cols[2].column, "amount");
    }

    #[tokio::test]
    async fn multiple_partitions_require_a_statement_handle() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                "resultSetMetaData": {
                    "rowType": [{ "name": "N", "type": "fixed" }],
                    "partitionInfo": [{ "rowCount": 1 }, { "rowCount": 1 }]
                },
                "data": [["1"]]
            })))
            .expect(1)
            .mount(&server)
            .await;
        let err = SnowflakeEngine::new(conn(&server))
            .execute("SELECT n FROM numbers")
            .await
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("multiple partitions without a statementHandle"),
            "{err}"
        );
    }

    #[tokio::test]
    async fn failed_or_incomplete_partitions_do_not_return_partial_rows() {
        for (response, expected_error) in [
            (ResponseTemplate::new(503), "503"),
            (compressed_partition(json!([])), "numRows reports 2"),
        ] {
            let server = MockServer::start().await;
            Mock::given(method("POST"))
                .and(path("/api/v2/statements"))
                .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                    "resultSetMetaData": {
                        "numRows": 2,
                        "rowType": [{ "name": "N", "type": "fixed" }],
                        "partitionInfo": [{ "rowCount": 1 }, { "rowCount": 1 }]
                    },
                    "data": [["1"]], "statementHandle": "h4"
                })))
                .expect(1)
                .mount(&server)
                .await;
            Mock::given(method("GET"))
                .and(path("/api/v2/statements/h4"))
                .and(query_param("partition", "1"))
                .respond_with(response)
                .expect(1)
                .mount(&server)
                .await;
            let err = SnowflakeEngine::new(conn(&server))
                .execute("SELECT n FROM numbers")
                .await
                .unwrap_err()
                .to_string();
            assert!(err.contains(expected_error), "{err}");
        }
    }

    /// 401/403 一类(auth、权限)也是 status 路径——message 字段照样送出。
    /// 与 422 的区别仅在状态码,调用者拿到的错误形态应保持一致(都是
    /// `Snowflake {status}: {message}`),便于上层按错误形态分类
    #[tokio::test]
    async fn auth_failure_body_is_surfaced() {
        for status in [401u16, 403, 503] {
            let server = MockServer::start().await;
            Mock::given(method("POST"))
                .and(path("/api/v2/statements"))
                .respond_with(ResponseTemplate::new(status).set_body_json(json!({
                    "code": "390100",
                    "message": format!("HTTP {status}: not authorized — token expired or missing role"),
                })))
                .mount(&server)
                .await;
            let err = SnowflakeEngine::new(conn(&server))
                .execute("SELECT 1")
                .await
                .unwrap_err()
                .to_string();
            assert!(err.contains(&status.to_string()), "{err}");
            assert!(err.contains("token expired"), "{err}");
        }
    }

    /// 5xx 一类(如 Snowflake 内部 504 / 503 + GW 错配)时 body 不是 JSON——
    /// 现有逻辑会尝试 JSON.parse 然后退回原文,所以「消息文案」仍能送达
    /// 调用者。把这层兜底路径钉住,免得被人「美化」掉
    #[tokio::test]
    async fn non_json_error_body_falls_back_to_plain_text() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(
                ResponseTemplate::new(500).set_body_string("Internal error: warehouse suspended\n"),
            )
            .mount(&server)
            .await;
        let err = SnowflakeEngine::new(conn(&server))
            .execute("SELECT 1")
            .await
            .unwrap_err()
            .to_string();
        assert!(err.contains("500"), "{err}");
        assert!(
            err.contains("warehouse suspended"),
            "非 JSON body 应原样透传,给上层一份能落库的描述:{err}"
        );
    }

    /// 2xx body 里若带非「成功」code(如 query canceled 取消信号),必须以业务
    /// 错误形态抛出,不让屏蔽在「看似 OK」的响应里——这是 Snowflake 的双层错误
    /// 模型,应当与 4xx/5xx 共用同一条 message 通道
    #[tokio::test]
    async fn business_error_code_in_a_2xx_body_is_bubbled_up() {
        let server = MockServer::start().await;
        Mock::given(method("POST"))
            .and(path("/api/v2/statements"))
            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
                "code": "000605",
                "message": "Statement was canceled by user.",
                // 没有 resultSetMetaData,与状态一致——成功响应必有 meta 或 data
            })))
            .mount(&server)
            .await;
        let err = SnowflakeEngine::new(conn(&server))
            .execute("SELECT 1")
            .await
            .unwrap_err()
            .to_string();
        assert!(err.contains("000605"), "{err}");
        assert!(err.contains("canceled"), "{err}");
    }
}