Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(influxdb): support table export query dispatch and pagination

t8y248410b70b4d3
diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs
index d5c8d4114a..6e0cbf9b5d 100644
--- a/crates/dbx-core/src/table_export.rs
+++ b/crates/dbx-core/src/table_export.rs
@@ -351,6 +351,11 @@ fn table_page_sql(
offset: u64,
batch_size: usize,
) -> String {
+ let default_order_columns = if sql_context.database_type == DatabaseType::InfluxDb {
+ primary_keys.iter().find(|column| column.eq_ignore_ascii_case("time")).map(std::slice::from_ref).unwrap_or(&[])
+ } else {
+ primary_keys
+ };
let sql = if use_keyset {
keyset_pagination_sql_with_identifier_quote(
col_names,
@@ -372,7 +377,7 @@ fn table_page_sql(
batch_size,
request.where_input.as_deref(),
request.order_by.as_deref(),
- primary_keys,
+ default_order_columns,
request.identifier_quote.as_deref(),
)
};
@@ -1472,8 +1477,10 @@ async fn export_table_data_core_inner(
// When no PK is available, falls back to offset-based pagination.
let has_custom_filter_or_order = request.where_input.as_ref().is_some_and(|value| !value.trim().is_empty())
|| request.order_by.as_ref().is_some_and(|value| !value.trim().is_empty());
- let use_keyset =
- !has_custom_filter_or_order && !primary_keys.is_empty() && primary_keys.iter().all(|pk| col_names.contains(pk));
+ let use_keyset = db_type != DatabaseType::InfluxDb
+ && !has_custom_filter_or_order
+ && !primary_keys.is_empty()
+ && primary_keys.iter().all(|pk| col_names.contains(pk));
// PK column indices within result rows (for extracting last-row values)
let pk_indices: Vec<usize> = if use_keyset {
@@ -2156,13 +2163,11 @@ async fn export_table_data_core_inner(
mod tests {
use super::*;
use crate::database_export::{clear_export_cancelled, set_export_cancelled};
- #[cfg(unix)]
use crate::models::connection::ConnectionConfig;
#[cfg(unix)]
use crate::plugins::{
InstalledPlugin, PluginDriverManifest, PluginDriverSession, PluginManifest, PluginRuntimeEnv,
};
- #[cfg(unix)]
use crate::storage::Storage;
use crate::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
use serde_json::json;
@@ -2310,6 +2315,149 @@ mod tests {
let _ = std::fs::remove_dir_all(fixture.dir);
}
+ async fn read_table_export_http_request(socket: &mut tokio::net::TcpStream) -> String {
+ use tokio::io::AsyncReadExt;
+
+ let mut request = Vec::new();
+ loop {
+ let mut chunk = [0_u8; 2048];
+ let bytes_read = socket.read(&mut chunk).await.unwrap();
+ assert!(bytes_read > 0, "request ended before headers were complete");
+ request.extend_from_slice(&chunk[..bytes_read]);
+ if request.windows(4).any(|window| window == b"\r\n\r\n") {
+ return String::from_utf8(request).unwrap();
+ }
+ }
+ }
+
+ async fn write_table_export_http_json(socket: &mut tokio::net::TcpStream, status: &str, body: &str) {
+ use tokio::io::AsyncWriteExt;
+
+ let response = format!(
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ socket.write_all(response.as_bytes()).await.unwrap();
+ }
+
+ fn table_export_query_param(target: &str, name: &str) -> Option<String> {
+ target.split_once('?')?.1.split('&').find_map(|pair| {
+ let (key, value) = pair.split_once('=')?;
+ (key == name).then(|| percent_encoding::percent_decode_str(value).decode_utf8_lossy().into_owned())
+ })
+ }
+
+ async fn spawn_influxdb_table_export_server() -> (String, tokio::task::JoinHandle<Vec<String>>) {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ let mut queries = Vec::new();
+ loop {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let request = read_table_export_http_request(&mut socket).await;
+ let target = request.lines().next().unwrap().split_whitespace().nth(1).unwrap();
+ let query = table_export_query_param(target, "q").expect("InfluxDB request must include q");
+ if query != "SHOW DATABASES" {
+ assert_eq!(table_export_query_param(target, "db").as_deref(), Some("dbx_issue_8022"));
+ }
+ let final_page = query.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2");
+ let body = match query.as_str() {
+ "SHOW DATABASES" => {
+ r#"{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["dbx_issue_8022"]]}]}]}"#
+ }
+ "SHOW TAG KEYS FROM \"weather\"" => {
+ r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["tagKey"],"values":[["location"]]}]}]}"#
+ }
+ "SHOW FIELD KEYS FROM \"weather\"" => {
+ r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["fieldKey","fieldType"],"values":[["temperature","float"]]}]}]}"#
+ }
+ value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 0") => {
+ r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:00:00Z","us-midwest",82]]}]}]}"#
+ }
+ value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 1") => {
+ r#"{"results":[{"statement_id":0,"series":[{"name":"weather","columns":["time","location","temperature"],"values":[["2023-09-03T12:01:00Z","us-midwest",83]]}]}]}"#
+ }
+ value if value.ends_with("ORDER BY \"time\" LIMIT 1 OFFSET 2") => {
+ r#"{"results":[{"statement_id":0}]}"#
+ }
+ unexpected => panic!("unexpected InfluxDB export query: {unexpected}"),
+ };
+ queries.push(query);
+ write_table_export_http_json(&mut socket, "200 OK", body).await;
+ if final_page {
+ break;
+ }
+ }
+ queries
+ });
+ (format!("http://{address}"), server)
+ }
+
+ #[tokio::test]
+ async fn influxdb_table_export_exports_paginated_rows() {
+ let (base_url, server) = spawn_influxdb_table_export_server().await;
+ let port = base_url.rsplit_once(':').unwrap().1.parse::<u16>().unwrap();
+ let dir = std::env::temp_dir().join(format!("dbx-influxdb-table-export-test-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let config: ConnectionConfig = serde_json::from_value(json!({
+ "id": "conn-influxdb-export",
+ "name": "InfluxDB export",
+ "db_type": "influxdb",
+ "host": "127.0.0.1",
+ "port": port,
+ "username": "",
+ "password": "",
+ "database": "dbx_issue_8022",
+ "query_timeout_secs": 30
+ }))
+ .unwrap();
+ let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ state.configs.write().await.insert(config.id.clone(), config);
+ let export_id = format!("export-{}", uuid::Uuid::new_v4());
+ let output = dir.join("weather.csv");
+ let request = TableExportRequest {
+ export_id,
+ connection_id: "conn-influxdb-export".to_string(),
+ database: "dbx_issue_8022".to_string(),
+ schema: None,
+ identifier_quote: None,
+ table_name: "weather".to_string(),
+ file_path: output.to_string_lossy().into_owned(),
+ format: "csv".to_string(),
+ csv_quote_mode: CsvQuoteMode::All,
+ columns: None,
+ column_types: None,
+ primary_keys: None,
+ where_input: None,
+ order_by: None,
+ skip_count: true,
+ batch_size: Some(1),
+ row_limit: None,
+ date_time_format: None,
+ numeric_column_right_align: false,
+ column_comments: None,
+ auto_filter: None,
+ };
+
+ export_table_data_core(&state, &request, |_| {}).await.unwrap();
+ let queries = tokio::time::timeout(std::time::Duration::from_secs(5), server)
+ .await
+ .expect("InfluxDB export server did not receive the final page")
+ .unwrap();
+ let csv = std::fs::read_to_string(&output).unwrap_or_default();
+ let _ = std::fs::remove_dir_all(&dir);
+
+ let first_page = queries.iter().find(|query| query.contains("LIMIT 1 OFFSET 0")).unwrap();
+ assert!(first_page.contains("ORDER BY \"time\""));
+ assert!(!first_page.contains("ORDER BY \"time\", \"location\""));
+ assert!(csv.contains("location"));
+ assert!(csv.contains("temperature"));
+ assert!(csv.contains("us-midwest"));
+ assert!(csv.contains("82"));
+ assert!(csv.contains("83"));
+ }
+
/// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer.
fn read_zip_entry(bytes: &[u8], path: &str) -> String {
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs
index 3a6b647c58..c3ef608c3e 100644
--- a/crates/dbx-core/src/transfer.rs
+++ b/crates/dbx-core/src/transfer.rs
@@ -5168,6 +5168,16 @@ async fn execute_on_pool_once(
let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
db::clickhouse_driver::execute_query_with_max_rows(&client, &database, sql, max_rows).await
}
+ PoolKind::InfluxDb(client) => {
+ let client = client.clone();
+ let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
+ db::influxdb_driver::execute_query(&client, &database, sql).await
+ }
+ PoolKind::InfluxDb3(client) => {
+ let client = client.clone();
+ let database = database_from_pool_key(pool_key).unwrap_or("default").to_string();
+ db::influxdb3_driver::execute_query(&client, &database, sql, max_rows).await
+ }
PoolKind::SqlServer(client) => {
let client = client.clone();
let mut client = client.lock().await;
@@ -8665,6 +8675,97 @@ mod tests {
(AppState::new(storage), dir)
}
+ async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) {
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ let (mut socket, _) = listener.accept().await.unwrap();
+ let mut request = Vec::new();
+ let header_end = loop {
+ let mut chunk = [0_u8; 2048];
+ let bytes_read = socket.read(&mut chunk).await.unwrap();
+ assert!(bytes_read > 0, "request ended before headers were complete");
+ request.extend_from_slice(&chunk[..bytes_read]);
+ if let Some(index) = request.windows(4).position(|window| window == b"\r\n\r\n") {
+ break index + 4;
+ }
+ };
+ let headers = String::from_utf8(request[..header_end].to_vec()).unwrap();
+ let content_length = headers
+ .lines()
+ .find_map(|line| {
+ let (name, value) = line.split_once(':')?;
+ name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::<usize>().unwrap())
+ })
+ .unwrap_or(0);
+ while request.len() < header_end + content_length {
+ let mut chunk = [0_u8; 2048];
+ let bytes_read = socket.read(&mut chunk).await.unwrap();
+ assert!(bytes_read > 0, "request ended before body was complete");
+ request.extend_from_slice(&chunk[..bytes_read]);
+ }
+ let body = r#"[{"time":"2026-09-03T00:00:00Z","value":42}]"#;
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ socket.write_all(response.as_bytes()).await.unwrap();
+ String::from_utf8(request).unwrap()
+ });
+ (format!("http://{address}"), server)
+ }
+
+ #[tokio::test]
+ async fn transfer_read_dispatches_influxdb3_with_database_and_row_limit() {
+ let (base_url, server) = spawn_influxdb3_transfer_server().await;
+ let (state, dir) = test_app_state().await;
+ let config: ConnectionConfig = serde_json::from_value(json!({
+ "id": "influxdb3-transfer",
+ "name": "InfluxDB 3 transfer",
+ "db_type": "influxdb3",
+ "host": "127.0.0.1",
+ "port": 8181,
+ "username": "",
+ "password": "",
+ "database": "metrics"
+ }))
+ .unwrap();
+ let client = db::influxdb3_driver::Influxdb3Client::new_for_config(
+ &base_url,
+ &config,
+ std::time::Duration::from_secs(5),
+ )
+ .unwrap();
+ state.configs.write().await.insert(config.id.clone(), config);
+ state
+ .update_connection_pools(|connections| {
+ connections.insert("influxdb3-transfer:metrics".to_string(), PoolKind::InfluxDb3(client));
+ })
+ .await;
+
+ let result = execute_read_on_pool_with_max_rows(
+ &state,
+ "influxdb3-transfer:metrics",
+ "SELECT value FROM weather",
+ Some(1),
+ )
+ .await
+ .unwrap();
+ let request = tokio::time::timeout(std::time::Duration::from_secs(5), server)
+ .await
+ .expect("InfluxDB 3 transfer server did not receive a request")
+ .unwrap();
+ let body = request.split_once("\r\n\r\n").unwrap().1;
+ let payload: serde_json::Value = serde_json::from_str(body).unwrap();
+ let _ = std::fs::remove_dir_all(dir);
+
+ assert_eq!(payload["db"], json!("metrics"));
+ assert_eq!(payload["q"], json!("SELECT value FROM weather LIMIT 2"));
+ assert_eq!(result.affected_rows, 1);
+ }
+
#[tokio::test]
async fn postgres_transfer_metadata_routes_agent_pools() {
let (state, dir) = test_app_state().await;