Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(import): preserve current timestamp expressions in sql imports

Xsyl.waya18bae6672d1
diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue
index 08b5f398e5..4e521daaef 100644
--- a/apps/desktop/src/components/import/TableImportDialog.vue
+++ b/apps/desktop/src/components/import/TableImportDialog.vue
@@ -18,6 +18,7 @@ import {
autoMapImportColumns,
buildTableImportParseOptions,
defaultTableImportEmptyStringAsNull,
+ formatTableImportPreviewCell,
formatTableImportElapsed,
nextTableImportWizardStep,
previousTableImportWizardStep,
@@ -973,12 +974,6 @@ function updateColumnDataType(sourceColumn: string, value: any) {
};
}
-function formatCell(value: unknown) {
- if (value === null) return "NULL";
- if (typeof value === "object") return JSON.stringify(value);
- return String(value);
-}
-
function goBack() {
wizardStep.value = previousTableImportWizardStep(wizardStep.value);
}
@@ -1836,7 +1831,7 @@ watch(rawProgressPercent, (percent) => {
<tbody>
<tr v-for="(row, rowIndex) in preview.rows" :key="rowIndex">
<td v-for="(cell, colIndex) in row" :key="colIndex" class="max-w-[180px] border-b border-r px-2 py-1.5 font-mono" :class="{ 'text-muted-foreground': cell === null }">
- <span class="block truncate">{{ formatCell(cell) }}</span>
+ <span class="block truncate">{{ formatTableImportPreviewCell(cell) }}</span>
</td>
</tr>
</tbody>
diff --git a/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts b/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts
index 55492d5cae..51b1454bc4 100644
--- a/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts
+++ b/apps/desktop/src/lib/__tests__/table/tableImport.spec.ts
@@ -4,6 +4,7 @@ import {
buildTableImportParseOptions,
defaultTableImportEmptyStringAsNull,
formatTableImportElapsed,
+ formatTableImportPreviewCell,
importDataTypeForDatabase,
nextTableImportWizardStep,
previousTableImportWizardStep,
@@ -70,6 +71,12 @@ describe("tableImport", () => {
expect(formatTableImportElapsed(61_000)).toBe("1m 1s");
});
+ it("shows SQL current timestamp markers without exposing internal JSON", () => {
+ expect(formatTableImportPreviewCell({ $dbxSqlImportExpression: "currentTimestamp" })).toBe("CURRENT_TIMESTAMP");
+ expect(formatTableImportPreviewCell({ $dbxSqlImportExpression: "other" })).toBe('{"$dbxSqlImportExpression":"other"}');
+ expect(formatTableImportPreviewCell({ value: 1 })).toBe('{"value":1}');
+ });
+
it("uses the backend elapsed time as the terminal source of truth", () => {
expect(resolveTableImportElapsed(1_500, 1_000, false)).toBe(1_500);
expect(resolveTableImportElapsed(1_500, 2_000, false)).toBe(2_000);
diff --git a/apps/desktop/src/lib/table/tableImport.ts b/apps/desktop/src/lib/table/tableImport.ts
index 3d92da2f91..8cf4ead86b 100644
--- a/apps/desktop/src/lib/table/tableImport.ts
+++ b/apps/desktop/src/lib/table/tableImport.ts
@@ -61,6 +61,16 @@ export function resolveTableImportElapsed(liveElapsedMs: number, backendElapsedM
return Math.max(live, backend ?? 0);
}
+export function formatTableImportPreviewCell(value: unknown): string {
+ if (value === null) return "NULL";
+ if (value !== null && typeof value === "object" && !Array.isArray(value)) {
+ const marker = value as Record<string, unknown>;
+ if (Object.keys(marker).length === 1 && marker.$dbxSqlImportExpression === "currentTimestamp") return "CURRENT_TIMESTAMP";
+ }
+ if (typeof value === "object") return JSON.stringify(value);
+ return String(value);
+}
+
export function tableImportProgressPercent(progress: TableImportProgressLike | null | undefined): number {
if (!progress) return 0;
if (progress.status === "done") return 100;
diff --git a/crates/dbx-core/src/data/cloudflare_d1/import.rs b/crates/dbx-core/src/data/cloudflare_d1/import.rs
index 3220b908df..e62df19b40 100644
--- a/crates/dbx-core/src/data/cloudflare_d1/import.rs
+++ b/crates/dbx-core/src/data/cloudflare_d1/import.rs
@@ -1,9 +1,18 @@
use crate::models::connection::DatabaseType;
-use crate::table_import::{mapping_indexes_for_columns, ImportSqlBatch, TableImportColumnMapping};
+use crate::table_import::{
+ mapping_indexes_for_columns, sql_import_is_current_timestamp_value, ImportSqlBatch, TableImportColumnMapping,
+};
use crate::transfer::generate_insert_typed;
use super::sql_limits::build_sql_batches;
+/// Sentinel used to carry the "current timestamp" import marker through
+/// `generate_insert_typed` (which only renders literals), replaced back with
+/// the CURRENT_TIMESTAMP keyword afterwards. Control-character framing keeps
+/// it from appearing in user data; if it ever does, we fail closed instead of
+/// silently rewriting the cell.
+const D1_CURRENT_TIMESTAMP_TOKEN: &str = "\u{1}dbx_current_timestamp\u{1}";
+
pub(crate) fn build_import_insert_batches(
rows: &[Vec<serde_json::Value>],
source_columns: &[String],
@@ -24,12 +33,25 @@ pub(crate) fn build_import_insert_batches(
.map(|(_, data_type)| data_type.clone())
})
.collect::<Vec<_>>();
+ if rows.iter().any(|row| {
+ row.iter()
+ .any(|value| matches!(value, serde_json::Value::String(text) if text.contains(D1_CURRENT_TIMESTAMP_TOKEN)))
+ }) {
+ return Err("import data contains a reserved expression token".to_string());
+ }
let rows = rows
.iter()
.map(|row| {
mapped
.iter()
- .map(|(source_index, _)| row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null))
+ .map(|(source_index, _)| {
+ let value = row.get(*source_index).cloned().unwrap_or(serde_json::Value::Null);
+ if sql_import_is_current_timestamp_value(&value) {
+ serde_json::Value::String(D1_CURRENT_TIMESTAMP_TOKEN.to_string())
+ } else {
+ value
+ }
+ })
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
@@ -38,7 +60,13 @@ pub(crate) fn build_import_insert_batches(
generate_insert_typed(&columns, &column_types, &rows[range], table, schema, &DatabaseType::CloudflareD1, None)
})
.map(|batches| {
- batches.into_iter().map(|batch| ImportSqlBatch { sql: batch.sql, row_count: batch.item_count }).collect()
+ batches
+ .into_iter()
+ .map(|batch| {
+ let sql = batch.sql.replace(&format!("'{}'", D1_CURRENT_TIMESTAMP_TOKEN), "CURRENT_TIMESTAMP");
+ ImportSqlBatch { sql, row_count: batch.item_count }
+ })
+ .collect()
})
}
@@ -91,4 +119,66 @@ mod tests {
assert_eq!(batch.sql.split(";\n").count(), 2);
assert!(batch.sql.split(";\n").all(|statement| statement.len() <= super::super::MAX_SQL_STATEMENT_BYTES));
}
+
+ #[test]
+ fn sql_import_current_timestamp_marker_becomes_the_keyword() {
+ let marker = serde_json::json!({ "$dbxSqlImportExpression": "currentTimestamp" });
+ let rows = vec![vec![serde_json::Value::from(1), marker]];
+ let columns = vec!["id".to_string(), "created_at".to_string()];
+ let mappings = vec![
+ TableImportColumnMapping {
+ source_column: "id".to_string(),
+ target_column: "id".to_string(),
+ target_data_type: None,
+ },
+ TableImportColumnMapping {
+ source_column: "created_at".to_string(),
+ target_column: "created_at".to_string(),
+ target_data_type: None,
+ },
+ ];
+
+ let batch = crate::table_import::build_import_insert_batch_from_rows(
+ &rows,
+ &columns,
+ &mappings,
+ &[],
+ "events",
+ "main",
+ &DatabaseType::CloudflareD1,
+ )
+ .unwrap()
+ .unwrap();
+
+ assert!(batch.sql.contains("CURRENT_TIMESTAMP"), "sql: {}", batch.sql);
+ assert!(!batch.sql.contains("$dbxSqlImportExpression"), "sql: {}", batch.sql);
+ assert!(!batch.sql.contains(D1_CURRENT_TIMESTAMP_TOKEN), "sql: {}", batch.sql);
+ }
+
+ #[test]
+ fn row_data_containing_the_sentinel_token_fails_closed() {
+ let rows = vec![vec![serde_json::Value::String(D1_CURRENT_TIMESTAMP_TOKEN.to_string())]];
+ let columns = vec!["payload".to_string()];
+ let mappings = vec![TableImportColumnMapping {
+ source_column: "payload".to_string(),
+ target_column: "payload".to_string(),
+ target_data_type: None,
+ }];
+
+ let result = crate::table_import::build_import_insert_batch_from_rows(
+ &rows,
+ &columns,
+ &mappings,
+ &[],
+ "events",
+ "main",
+ &DatabaseType::CloudflareD1,
+ );
+
+ assert!(
+ result.is_err(),
+ "expected reserved-token rejection, got: {:?}",
+ result.ok().flatten().map(|batch| batch.sql)
+ );
+ }
}
diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs
index 416954d074..35d30fb86a 100644
--- a/crates/dbx-core/src/data/table_import.rs
+++ b/crates/dbx-core/src/data/table_import.rs
@@ -75,6 +75,8 @@ const POSTGRES_COPY_MAX_ROWS: usize = 50_000;
// NVARCHAR staging values. The source JSON batch remains owned by the parser;
// the bulk path must never add a second, batch-sized string matrix beside it.
const SQLSERVER_BULK_ROW_MEMORY_BYTES: usize = 16 * 1024 * 1024;
+const SQL_IMPORT_EXPRESSION_KEY: &str = "$dbxSqlImportExpression";
+const SQL_IMPORT_CURRENT_TIMESTAMP: &str = "currentTimestamp";
pub fn table_import_client_session_id(import_id: &str) -> String {
task_client_session_id("table-import", import_id)
@@ -1833,6 +1835,43 @@ fn sql_import_number_value(raw: &str, context: &str) -> Result<serde_json::Value
Err(format!("{context}: unsupported numeric literal '{raw}'"))
}
+// SQL imports round-trip rows through the preview and prepared-source JSON. Keep
+// allowlisted server-evaluated expressions as a fixed marker so the SQL builder
+// can emit only its own canonical expression, never arbitrary frontend text.
+fn sql_import_current_timestamp_value() -> serde_json::Value {
+ serde_json::json!({ SQL_IMPORT_EXPRESSION_KEY: SQL_IMPORT_CURRENT_TIMESTAMP })
+}
+
+pub(crate) fn sql_import_is_current_timestamp_value(value: &serde_json::Value) -> bool {
+ value.as_object().is_some_and(|object| {
+ object.len() == 1
+ && object.get(SQL_IMPORT_EXPRESSION_KEY).and_then(serde_json::Value::as_str)
+ == Some(SQL_IMPORT_CURRENT_TIMESTAMP)
+ })
+}
+
+fn sql_import_is_current_timestamp_function(function: &sqlparser::ast::Function) -> bool {
+ let name = function.name.to_string().to_ascii_lowercase();
+ if !matches!(name.as_str(), "now" | "sysdate" | "current_timestamp" | "localtime" | "localtimestamp") {
+ return false;
+ }
+ match &function.args {
+ FunctionArguments::None => true,
+ FunctionArguments::List(list) => {
+ list.args.is_empty() && list.clauses.is_empty() && list.duplicate_treatment.is_none()
+ }
+ _ => false,
+ }
+}
+
+fn sql_import_current_timestamp_sql(db_type: &DatabaseType) -> &'static str {
+ if *db_type == DatabaseType::ClickHouse {
+ "now()"
+ } else {
+ "CURRENT_TIMESTAMP"
+ }
+}
+
fn sql_import_expr_value(expr: &Expr, context: &str) -> Result<serde_json::Value, String> {
match expr {
Expr::Value(value) => match &value.value {
@@ -1875,7 +1914,13 @@ fn sql_import_expr_value(expr: &Expr, context: &str) -> Result<serde_json::Value
// `DATE '...'`、`TIMESTAMP '...'`、`TIME '...'` 等标准字面量:值就是引号里的字符串,无损导入。
Expr::TypedString(typed) => sql_import_typed_string(typed, context),
// 白名单字面量日期函数(TO_DATE/TO_TIMESTAMP/...):参数全为字符串字面量时按第一个参数导入。
- Expr::Function(function) => sql_import_literal_temporal_function(function, context),
+ Expr::Function(function) => {
+ if sql_import_is_current_timestamp_function(function) {
+ Ok(sql_import_current_timestamp_value())
+ } else {
+ sql_import_literal_temporal_function(function, context)
+ }
+ }
_ => Err(sql_import_unsupported_expression(context)),
}
}
@@ -4881,6 +4926,10 @@ fn import_value_rows_sql(
}
let source_value = row.get(*source_index).unwrap_or(&serde_json::Value::Null);
let data_type = plan.column_types.get(target_index).and_then(|data_type| data_type.as_deref());
+ if sql_import_is_current_timestamp_value(source_value) {
+ values.push_str(sql_import_current_timestamp_sql(db_type));
+ continue;
+ }
let normalized =
normalize_import_value_cow(source_value, data_type, db_type, kingbase_oracle_mode, date_time_format);
values.push_str(&escape_value_typed(normalized.as_ref(), db_type, data_type));
@@ -5254,6 +5303,9 @@ fn infer_string_type(value: &str) -> ImportInferredType {
}
fn infer_value_type(value: &serde_json::Value) -> Option<ImportInferredType> {
+ if sql_import_is_current_timestamp_value(value) {
+ return Some(ImportInferredType::Timestamp);
+ }
match value {
serde_json::Value::Null => None,
serde_json::Value::Bool(_) => Some(ImportInferredType::Boolean),
@@ -10553,6 +10605,54 @@ mod tests {
);
}
+ #[test]
+ fn sql_import_preserves_current_timestamp_functions() {
+ let options = sql_import_options(DatabaseType::Mysql);
+ let script = b"INSERT INTO sai_user (id, created_at, updated_at) VALUES (1, NOW(), SYSDATE());";
+ let parsed = parse_sql_bytes_with_options(script, &options, 10).unwrap();
+
+ assert_eq!(
+ parsed.rows[0],
+ vec![
+ serde_json::json!(1),
+ serde_json::json!({ "$dbxSqlImportExpression": "currentTimestamp" }),
+ serde_json::json!({ "$dbxSqlImportExpression": "currentTimestamp" }),
+ ]
+ );
+ }
+
+ #[test]
+ fn sql_import_emits_current_timestamp_without_quoting_the_marker() {
+ let options = sql_import_options(DatabaseType::Mysql);
+ let parsed =
+ parse_sql_bytes_with_options(b"INSERT INTO sai_user (id, created_at) VALUES (1, NOW());", &options, 10)
+ .unwrap();
+ let mappings = parsed
+ .columns
+ .iter()
+ .map(|column| TableImportColumnMapping {
+ source_column: column.clone(),
+ target_column: column.clone(),
+ target_data_type: None,
+ })
+ .collect::<Vec<_>>();
+ let target_types =
+ vec![("id".to_string(), "bigint".to_string()), ("created_at".to_string(), "datetime".to_string())];
+
+ let batches =
+ build_import_insert_batches(&parsed, &mappings, &target_types, "sai_user", "", &DatabaseType::Mysql, 500)
+ .unwrap();
+
+ assert_eq!(batches.len(), 1);
+ let generated_sql = batches[0].sql.replace('\n', " ");
+ assert!(
+ generated_sql.contains("VALUES (1, CURRENT_TIMESTAMP)"),
+ "generated SQL should preserve the server-side time expression: {}",
+ batches[0].sql
+ );
+ assert!(!batches[0].sql.contains(SQL_IMPORT_EXPRESSION_KEY));
+ }
+
#[test]
fn sql_import_rejects_replace() {
let error = parse_sql_bytes(b"REPLACE INTO users (id) VALUES (1);", 10).unwrap_err();
@@ -10561,7 +10661,7 @@ mod tests {
#[test]
fn sql_import_rejects_expressions() {
- let error = parse_sql_bytes(b"INSERT INTO t (a) VALUES (NOW());", 10).unwrap_err();
+ let error = parse_sql_bytes(b"INSERT INTO t (a) VALUES (UUID());", 10).unwrap_err();
assert!(error.contains("not supported"));
}
@@ -10606,9 +10706,9 @@ mod tests {
#[test]
fn sql_import_rejects_unlisted_function_still() {
- // 非白名单函数(如 NOW())仍按表达式拒绝。
+ // 非白名单函数(如 RAND())仍按表达式拒绝。
let options = sql_import_options(DatabaseType::Mysql);
- let error = parse_sql_bytes_with_options(b"INSERT INTO t (a) VALUES (NOW());", &options, 10).unwrap_err();
+ let error = parse_sql_bytes_with_options(b"INSERT INTO t (a) VALUES (RAND());", &options, 10).unwrap_err();
assert!(error.contains("not supported"));
}