Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

17.1 KB
Raw Blame
use std::collections::HashSet;

use deadpool_postgres::Pool;

use super::{ObjectInfo, TableInfo};
use crate::db;

const CLOUD_BERRY_TABLE_DDL_SQL: &str = "SELECT pg_get_tabledef($1, $2, true)";

const CLOUD_BERRY_EXTERNAL_TABLES_SQL: &str = "SELECT c.relname \
     FROM pg_catalog.pg_class c \
     JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \
     WHERE n.nspname = $1 AND c.relkind = 'f' AND c.relname = ANY($2::text[])";

const CLOUD_BERRY_TABLE_MODIFIERS_SQL: &str = "SELECT COALESCE(am.amname, '')::text AS access_method, \
            COALESCE(array_to_string(c.reloptions, E'\\n'), '')::text AS reloptions, \
            COALESCE(dp.policytype::text, '')::text AS policy_type, \
            COALESCE(string_agg(a.attname, E'\\n' \
              ORDER BY array_position(dp.distkey::smallint[], a.attnum::smallint)), '')::text \
              AS distribution_columns, \
            bool_or(c.relkind = 'f') AS is_external, \
            COALESCE(fs.srvname, '')::text AS external_server, \
            ft.ftoptions AS external_options \
     FROM pg_catalog.pg_class c \
     JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \
     LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam \
     LEFT JOIN pg_catalog.gp_distribution_policy dp ON dp.localoid = c.oid \
     LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid AND a.attnum = ANY(dp.distkey) \
     LEFT JOIN pg_catalog.pg_foreign_table ft ON ft.ftrelid = c.oid \
     LEFT JOIN pg_catalog.pg_foreign_server fs ON fs.oid = ft.ftserver \
     WHERE n.nspname = $1 AND c.relname = $2 \
       AND c.relkind IN ('r', 'p', 'f') \
     GROUP BY c.oid, am.amname, c.reloptions, dp.policytype, dp.distkey, fs.srvname, ft.ftoptions \
     LIMIT 1";

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DistributionPolicy {
    Hash(Vec<String>),
    Random,
    Replicated,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExternalTableDefinition {
    pub server: String,
    pub options: Vec<String>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableModifiers {
    pub access_method: Option<String>,
    pub reloptions: Vec<String>,
    pub distribution: Option<DistributionPolicy>,
    pub external: Option<ExternalTableDefinition>,
}

pub async fn list_tables_filtered(
    pool: &Pool,
    schema: &str,
    filter: Option<&str>,
    limit: Option<usize>,
    offset: Option<usize>,
) -> Result<Vec<TableInfo>, String> {
    let mut tables = db::postgres::list_tables_filtered(pool, schema, filter, limit, offset).await?;
    annotate_external_tables(pool, schema, &mut tables).await;
    Ok(tables)
}

pub async fn list_objects(pool: &Pool, schema: &str) -> Result<Vec<ObjectInfo>, String> {
    let mut objects = db::postgres::list_objects(pool, schema, true, true, false).await?;
    let names = objects.iter().map(|object| object.name.clone()).collect::<Vec<_>>();
    let external_names = external_table_names(pool, schema, &names).await.unwrap_or_else(|error| {
        log::debug!("[cloudberry][list_objects:external-table-fallback] error={error}");
        HashSet::new()
    });
    for object in &mut objects {
        if external_names.contains(&object.name) {
            object.object_type = "EXTERNAL TABLE".to_string();
        }
    }
    Ok(objects)
}

pub async fn table_ddl(pool: &Pool, schema: &str, table: &str) -> Result<String, String> {
    let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?;
    let row = client
        .query_opt(CLOUD_BERRY_TABLE_DDL_SQL, &[&schema, &table])
        .await
        .map_err(|error| error.to_string())?
        .ok_or_else(|| format!("Cloudberry table not found: {schema}.{table}"))?;
    let ddl = row.try_get::<_, Option<String>>(0).map_err(|error| error.to_string())?.unwrap_or_default();
    normalize_ddl(ddl)
}

pub async fn table_modifiers(pool: &Pool, schema: &str, table: &str) -> Result<TableModifiers, String> {
    let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?;
    let row = client
        .query_opt(CLOUD_BERRY_TABLE_MODIFIERS_SQL, &[&schema, &table])
        .await
        .map_err(|error| error.to_string())?
        .ok_or_else(|| format!("Cloudberry table not found: {schema}.{table}"))?;

    let access_method = non_empty(row.try_get::<_, String>(0).map_err(|error| error.to_string())?)
        .filter(|method| !method.eq_ignore_ascii_case("heap"));
    let reloptions = split_catalog_lines(row.try_get::<_, String>(1).map_err(|error| error.to_string())?);
    let policy_type = row.try_get::<_, String>(2).map_err(|error| error.to_string())?;
    let distribution_columns = split_catalog_lines(row.try_get::<_, String>(3).map_err(|error| error.to_string())?);
    let distribution = match policy_type.as_str() {
        "r" => Some(DistributionPolicy::Replicated),
        "p" if distribution_columns.is_empty() => Some(DistributionPolicy::Random),
        "p" => Some(DistributionPolicy::Hash(distribution_columns)),
        _ => None,
    };

    let is_external = row.try_get::<_, bool>(4).map_err(|error| error.to_string())?;
    let external = if is_external {
        let server = row.try_get::<_, String>(5).map_err(|error| error.to_string())?;
        if server.trim().is_empty() {
            return Err(format!("Cloudberry external table has no foreign server: {schema}.{table}"));
        }
        Some(ExternalTableDefinition {
            server,
            options: row.try_get::<_, Option<Vec<String>>>(6).map_err(|error| error.to_string())?.unwrap_or_default(),
        })
    } else {
        None
    };

    Ok(TableModifiers { access_method, reloptions, distribution, external })
}

pub fn append_table_modifiers(ddl: &str, modifiers: &TableModifiers) -> Result<String, String> {
    if let Some(external) = modifiers.external.as_ref() {
        return render_external_table_ddl(ddl, external);
    }

    let clauses = render_table_modifier_clauses(modifiers);
    if clauses.is_empty() {
        return Ok(ddl.to_string());
    }
    let insertion = ddl
        .find(";\n")
        .or_else(|| ddl.find(';'))
        .ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?;
    let mut output = String::with_capacity(ddl.len() + clauses.len() + 2);
    output.push_str(&ddl[..insertion]);
    output.push('\n');
    output.push_str(&clauses);
    output.push_str(&ddl[insertion..]);
    Ok(output)
}

fn render_external_table_ddl(ddl: &str, external: &ExternalTableDefinition) -> Result<String, String> {
    // The fallback DDL may already read `CREATE FOREIGN TABLE` — a foreign
    // table's own relkind ('f') makes the generic renderer emit that header
    // directly — or plain `CREATE TABLE` from an older/other rendering path.
    // Accept either so this doesn't regress when the upstream header changes.
    let create_foreign_table = "CREATE FOREIGN TABLE ";
    let create_table = "CREATE TABLE ";
    let header_len = if ddl.starts_with(create_foreign_table) {
        create_foreign_table.len()
    } else if ddl.starts_with(create_table) {
        create_table.len()
    } else {
        return Err("Cloudberry external-table fallback expected CREATE TABLE DDL".to_string());
    };
    let insertion = ddl
        .find(";\n")
        .or_else(|| ddl.find(';'))
        .ok_or_else(|| "Cloudberry fallback DDL has no CREATE TABLE terminator".to_string())?;
    // When the fallback DDL already read `CREATE FOREIGN TABLE`, the generic
    // renderer also already appended its own ` SERVER ... [OPTIONS (...)]`
    // clause (from the same relkind='f' catalog data) right after the column
    // list's closing paren. Drop that so this doesn't emit the clause twice —
    // Cloudberry's own `external` definition above is authoritative here.
    // Locate the column list's own closing paren structurally (tracking
    // parenthesis depth and skipping quoted content) instead of searching for
    // the literal text `" SERVER \""`, which a column's DEFAULT/CHECK
    // expression could contain and match spuriously.
    let body_end = super::ddl_scan::find_top_level_paren_close(ddl, header_len).unwrap_or(insertion);
    let mut output = String::with_capacity(ddl.len() + external.options.len() * 24 + 48);
    output.push_str("CREATE FOREIGN TABLE ");
    output.push_str(&ddl[header_len..body_end]);
    output.push_str("\nSERVER ");
    output.push_str(&db::postgres::pg_quote_ident(&external.server));
    if !external.options.is_empty() {
        // Cloudberry 2.x stores external tables as foreign tables. Reusing the
        // server options preserves URI, format and execution-location details.
        output.push_str("\nOPTIONS (\n  ");
        output.push_str(
            &external
                .options
                .iter()
                .map(|option| render_foreign_table_option(option))
                .collect::<Result<Vec<_>, _>>()?
                .join(",\n  "),
        );
        output.push_str("\n)");
    }
    output.push_str(&ddl[insertion..]);
    Ok(output)
}

fn render_foreign_table_option(option: &str) -> Result<String, String> {
    let (name, value) =
        option.split_once('=').ok_or_else(|| format!("Invalid Cloudberry foreign-table option: {option}"))?;
    Ok(format!("{} {}", db::postgres::pg_quote_ident(name), quote_sql_string(value)))
}

fn render_table_modifier_clauses(modifiers: &TableModifiers) -> String {
    let mut clauses = Vec::new();
    if let Some(access_method) = modifiers.access_method.as_deref() {
        clauses.push(format!("USING {}", db::postgres::pg_quote_ident(access_method)));
    }
    if !modifiers.reloptions.is_empty() {
        clauses.push(format!("WITH (\n  {}\n)", modifiers.reloptions.join(",\n  ")));
    }
    if let Some(distribution) = modifiers.distribution.as_ref() {
        clauses.push(match distribution {
            DistributionPolicy::Hash(columns) => format!(
                "DISTRIBUTED BY ({})",
                columns.iter().map(|column| db::postgres::pg_quote_ident(column)).collect::<Vec<_>>().join(", ")
            ),
            DistributionPolicy::Random => "DISTRIBUTED RANDOMLY".to_string(),
            DistributionPolicy::Replicated => "DISTRIBUTED REPLICATED".to_string(),
        });
    }
    clauses.join("\n")
}

async fn annotate_external_tables(pool: &Pool, schema: &str, tables: &mut [TableInfo]) {
    let names = tables.iter().map(|table| table.name.clone()).collect::<Vec<_>>();
    let external_names = external_table_names(pool, schema, &names).await.unwrap_or_else(|error| {
        log::debug!("[cloudberry][list_tables:external-table-fallback] error={error}");
        HashSet::new()
    });
    for table in tables {
        if external_names.contains(&table.name) {
            table.table_type = "EXTERNAL TABLE".to_string();
        }
    }
}

async fn external_table_names(pool: &Pool, schema: &str, names: &[String]) -> Result<HashSet<String>, String> {
    if names.is_empty() {
        return Ok(HashSet::new());
    }
    let client = db::postgres::checkout_postgres_client(pool, None, super::connection_timeout()).await?;
    let rows =
        client.query(CLOUD_BERRY_EXTERNAL_TABLES_SQL, &[&schema, &names]).await.map_err(|error| error.to_string())?;
    Ok(rows.into_iter().filter_map(|row| row.try_get::<_, String>(0).ok()).collect())
}

fn normalize_ddl(ddl: String) -> Result<String, String> {
    let ddl = ddl.trim();
    if ddl.is_empty() {
        return Err("Cloudberry returned an empty table DDL".to_string());
    }
    if ddl.ends_with(';') {
        Ok(format!("{ddl}\n"))
    } else {
        Ok(format!("{ddl};\n"))
    }
}

fn non_empty(value: String) -> Option<String> {
    let value = value.trim();
    (!value.is_empty()).then(|| value.to_string())
}

fn split_catalog_lines(value: String) -> Vec<String> {
    value.lines().map(str::trim).filter(|value| !value.is_empty()).map(str::to_string).collect()
}

fn quote_sql_string(value: &str) -> String {
    format!("'{}'", value.replace('\'', "''"))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn modifiers(distribution: Option<DistributionPolicy>) -> TableModifiers {
        TableModifiers { access_method: None, reloptions: Vec::new(), distribution, external: None }
    }

    #[test]
    fn appends_hash_distribution_before_table_terminator() {
        let ddl = "CREATE TABLE \"public\".\"events\" (\n  \"tenant_id\" integer\n);\n";
        let rendered = append_table_modifiers(
            ddl,
            &modifiers(Some(DistributionPolicy::Hash(vec!["tenant_id".to_string(), "event id".to_string()]))),
        )
        .unwrap();

        assert_eq!(
            rendered,
            "CREATE TABLE \"public\".\"events\" (\n  \"tenant_id\" integer\n)\nDISTRIBUTED BY (\"tenant_id\", \"event id\");\n"
        );
    }

    #[test]
    fn appends_storage_and_replicated_distribution() {
        let ddl = "CREATE TABLE \"public\".\"dimensions\" (\n  \"id\" integer\n);\n";
        let rendered = append_table_modifiers(
            ddl,
            &TableModifiers {
                access_method: Some("ao_column".to_string()),
                reloptions: vec!["compresstype=zstd".to_string(), "compresslevel=3".to_string()],
                distribution: Some(DistributionPolicy::Replicated),
                external: None,
            },
        )
        .unwrap();

        assert!(rendered.contains("USING \"ao_column\""));
        assert!(rendered.contains("WITH (\n  compresstype=zstd,\n  compresslevel=3\n)"));
        assert!(rendered.contains("DISTRIBUTED REPLICATED;"));
    }

    #[test]
    fn renders_external_table_from_foreign_options() {
        let ddl = "CREATE TABLE \"public\".\"external_events\" (\n  \"id\" integer\n);\n";
        let rendered = append_table_modifiers(
            ddl,
            &TableModifiers {
                external: Some(ExternalTableDefinition {
                    server: "gp_exttable_server".to_string(),
                    options: vec![
                        "format=csv".to_string(),
                        "location_uris=file://cdw/tmp/events.csv".to_string(),
                        "null=".to_string(),
                    ],
                }),
                ..modifiers(None)
            },
        )
        .unwrap();

        assert!(rendered.starts_with("CREATE FOREIGN TABLE \"public\".\"external_events\""));
        assert!(rendered.contains("SERVER \"gp_exttable_server\""));
        assert!(rendered.contains("\"location_uris\" 'file://cdw/tmp/events.csv'"));
        assert!(rendered.contains("\"null\" ''"));
    }

    #[test]
    fn renders_external_table_from_already_foreign_ddl_without_duplicate_server_clause() {
        // The generic renderer now derives `is_foreign` straight from relkind
        // and may hand back a `CREATE FOREIGN TABLE ... SERVER ...` fallback
        // DDL directly, not just plain `CREATE TABLE`.
        let ddl =
            "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n  \"id\" integer\n) SERVER \"stale_server\";\n";
        let rendered = append_table_modifiers(
            ddl,
            &TableModifiers {
                external: Some(ExternalTableDefinition {
                    server: "gp_exttable_server".to_string(),
                    options: vec!["format=csv".to_string()],
                }),
                ..modifiers(None)
            },
        )
        .unwrap();

        assert!(rendered.starts_with("CREATE FOREIGN TABLE \"public\".\"external_events\""));
        assert_eq!(rendered.matches("SERVER").count(), 1);
        assert!(rendered.contains("SERVER \"gp_exttable_server\""));
        assert!(!rendered.contains("stale_server"));
    }

    #[test]
    fn renders_external_table_when_a_column_default_contains_literal_server_text() {
        // A naive text search for `" SERVER \""` to find the stale SERVER
        // clause's start would match this DEFAULT's embedded text instead of
        // the real trailing SERVER clause, truncating the column list.
        let ddl = "CREATE FOREIGN TABLE \"public\".\"external_events\" (\n  \"id\" integer,\n  \"note\" text DEFAULT 'contact SERVER \"admin\"'\n) SERVER \"stale_server\";\n";
        let rendered = append_table_modifiers(
            ddl,
            &TableModifiers {
                external: Some(ExternalTableDefinition {
                    server: "gp_exttable_server".to_string(),
                    options: vec!["format=csv".to_string()],
                }),
                ..modifiers(None)
            },
        )
        .unwrap();

        assert!(rendered.contains("\"note\" text DEFAULT 'contact SERVER \"admin\"'"));
        assert_eq!(
            rendered.matches("SERVER").count(),
            2,
            "expected exactly the embedded literal plus the real clause: {rendered}"
        );
        assert!(rendered.contains("SERVER \"gp_exttable_server\""));
        assert!(!rendered.contains("stale_server"));
    }

    #[test]
    fn ddl_queries_avoid_privileged_external_catalog_view() {
        assert_eq!(CLOUD_BERRY_TABLE_DDL_SQL, "SELECT pg_get_tabledef($1, $2, true)");
        assert!(CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("gp_distribution_policy"));
        assert!(CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("c.relkind = 'f'"));
        assert!(CLOUD_BERRY_EXTERNAL_TABLES_SQL.contains("c.relkind = 'f'"));
        assert!(!CLOUD_BERRY_TABLE_MODIFIERS_SQL.contains("pg_exttable"));
        assert!(!CLOUD_BERRY_EXTERNAL_TABLES_SQL.contains("pg_exttable"));
    }
}