25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
fix(import): honor custom title rows in Excel streaming import
lxk955fa7c9ef49e19
diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rsindex 7ec866e8fb..3344b7ea6c 100644--- a/crates/dbx-core/src/data/table_import.rs+++ b/crates/dbx-core/src/data/table_import.rs@@ -3462,6 +3462,7 @@ fn parse_xlsx_preview_file_with_options(let date_1904 = xlsx_workbook_uses_1904_date_system(&workbook_xml);let empty_string_as_null = options.empty_string_as_null.unwrap_or(true);let row_range = effective_import_row_range(options)?;+ let min_import_row = row_range.title_row.unwrap_or(row_range.data_start_row);let preview_limit = preview_limit.max(1);let preview_end_from = |first_row: usize| {let preview_last_row = first_row.saturating_add(preview_limit.saturating_sub(1));@@ -3506,7 +3507,9 @@ fn parse_xlsx_preview_file_with_options({break;}- observed_max_row = observed_max_row.max(current_row);+ if current_row >= min_import_row {+ observed_max_row = observed_max_row.max(current_row);+ }}Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {let position = xml_attr_value(&reader, &element, b"r")@@ -3515,13 +3518,14 @@ fn parse_xlsx_preview_file_with_options(.unwrap_or_else(|| (current_row.max(1), current_column.saturating_add(1).max(1)));current_row = position.0;current_column = position.1;- observed_min_row = observed_min_row.min(position.0);- requested_last_row = preview_end_from(row_range.data_start_row.max(observed_min_row));- observed_min_column = observed_min_column.min(position.1);- observed_max_row = observed_max_row.max(position.0);- observed_max_column = observed_max_column.max(position.1);+ if current_row >= min_import_row {+ observed_min_row = observed_min_row.min(position.0);+ requested_last_row = preview_end_from(row_range.data_start_row.max(observed_min_row));+ observed_min_column = observed_min_column.min(position.1);+ observed_max_row = observed_max_row.max(position.0);+ observed_max_column = observed_max_column.max(position.1);+ }}- Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => continue,Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {let position = xml_attr_value(&reader, &element, b"r").as_deref()@@ -3570,15 +3574,17 @@ fn parse_xlsx_preview_file_with_options(}Ok(Event::End(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {if let Some((row, column)) = current_position.take() {- observed_min_row = observed_min_row.min(row);- requested_last_row = preview_end_from(row_range.data_start_row.max(observed_min_row));- observed_min_column = observed_min_column.min(column);- observed_max_column = observed_max_column.max(column);- observed_max_row = observed_max_row.max(row);- if row == row_range.title_row.unwrap_or_default()- || (row >= row_range.data_start_row && row <= requested_last_row)- {- raw_cells.insert((row, column), std::mem::take(&mut current_cell));+ if row >= min_import_row {+ observed_min_row = observed_min_row.min(row);+ requested_last_row = preview_end_from(row_range.data_start_row.max(observed_min_row));+ observed_min_column = observed_min_column.min(column);+ observed_max_column = observed_max_column.max(column);+ observed_max_row = observed_max_row.max(row);+ if row == row_range.title_row.unwrap_or_default()+ || (row >= row_range.data_start_row && row <= requested_last_row)+ {+ raw_cells.insert((row, column), std::mem::take(&mut current_cell));+ }}}inline_phonetic_depth = 0;@@ -3616,8 +3622,10 @@ fn parse_xlsx_preview_file_with_options());}let dimension_end_column = dimension- .filter(|((dimension_start_row, dimension_start_column), _)| {- *dimension_start_row == start_row && *dimension_start_column == start_column+ .filter(|((dimension_start_row, dimension_start_column), (dimension_end_row, _))| {+ *dimension_start_row <= start_row+ && start_row <= *dimension_end_row+ && *dimension_start_column == start_column}).map(|(_, (_, end_column))| end_column).filter(|end_column| {@@ -3877,6 +3885,7 @@ struct XlsxStreamRowsState {start_row: Option<usize>,start_column: usize,declared_column_count: Option<usize>,+ has_expected_columns: bool,columns: Vec<String>,header_sent: bool,pending_rows: Vec<Vec<serde_json::Value>>,@@ -3896,6 +3905,7 @@ impl XlsxStreamRowsState {batch_size: usize,) -> Self {let batch_size = batch_size.max(1);+ let has_expected_columns = expected_columns.is_some();Self {sender,row_range,@@ -3903,6 +3913,7 @@ impl XlsxStreamRowsState {start_row: None,start_column: 0,declared_column_count: None,+ has_expected_columns,columns: expected_columns.unwrap_or_default(),header_sent: false,pending_rows: Vec::with_capacity(batch_size),@@ -3922,8 +3933,10 @@ impl XlsxStreamRowsState {let dimension = self.dimension.filter(|((start_row, start_column), (end_row, end_column))| {let column_count = end_column.saturating_sub(*start_column).saturating_add(1);let row_count = end_row.saturating_sub(*start_row).saturating_add(1);- *start_row == first_row- && *start_column == first_column+ *start_row <= first_row+ && first_row <= *end_row+ && *start_column <= first_column+ && first_column <= *end_column&& column_count <= MAX_FAST_PREVIEW_CELLS&& expected_column_count.map_or(column_count.saturating_mul(row_count) <= MAX_FAST_PREVIEW_CELLS, |expected| {@@ -3931,11 +3944,39 @@ impl XlsxStreamRowsState {})});self.start_row = Some(first_row);- self.start_column = first_column;+ self.start_column =+ dimension.map(|((_, start_column), _)| start_column.min(first_column)).unwrap_or(first_column);self.declared_column_count = dimension.map(|((_, start_column), (_, end_column))| end_column.saturating_sub(start_column).saturating_add(1));}+ fn ensure_column_range(&mut self, absolute_row: usize, absolute_column: usize) -> Result<(), String> {+ self.initialize_range(absolute_row, absolute_column);+ if absolute_column < self.start_column {+ if self.header_sent {+ return Err(format!(+ "Excel row {absolute_row} contains a cell before the confirmed import range start column"+ ));+ }+ let shift = self.start_column.saturating_sub(absolute_column);+ self.start_column = absolute_column;+ if shift > 0 {+ let mut shifted = vec![serde_json::Value::Null; shift];+ shifted.append(&mut self.current_values);+ self.current_values = shifted;+ if !self.has_expected_columns && !self.columns.is_empty() {+ let mut shifted_columns: Vec<String> = (1..=shift).map(|index| format!("column_{index}")).collect();+ shifted_columns.append(&mut self.columns);+ self.columns = unique_import_headers(shifted_columns);+ }+ if let Some(count) = self.declared_column_count.as_mut() {+ *count = count.saturating_add(shift);+ }+ }+ }+ Ok(())+ }+fn selected_range_finished(&self, absolute_row: usize) -> bool {self.row_range.last_data_row.is_some_and(|last| absolute_row > last)}@@ -3946,7 +3987,7 @@ impl XlsxStreamRowsState {absolute_column: usize,text_source_columns: &HashSet<String>,) -> bool {- self.initialize_range(absolute_row, absolute_column);+ let _ = self.ensure_column_range(absolute_row, absolute_column);absolute_column.checked_sub(self.start_column).and_then(|offset| self.columns.get(offset))@@ -3961,15 +4002,13 @@ impl XlsxStreamRowsState {has_content: bool,progress: u64,) -> Result<(), String> {- self.initialize_range(absolute_row, absolute_column);if self.current_row != Some(absolute_row) {self.flush_current_row(progress)?;self.current_row = Some(absolute_row);}+ self.ensure_column_range(absolute_row, absolute_column)?;self.current_row_has_content |= has_content;- let column_offset = absolute_column.checked_sub(self.start_column).ok_or_else(|| {- format!("Excel row {absolute_row} contains a cell before the detected import range start column")- })?;+ let column_offset = absolute_column.saturating_sub(self.start_column);if column_offset >= MAX_FAST_PREVIEW_CELLS {return Err(format!("Excel import column {} exceeds the safety limit", column_offset + 1));}@@ -4160,6 +4199,7 @@ fn stream_xlsx_rows_to_channel_with_control(&mut on_shared_progress,)?;let row_range = effective_import_row_range(options)?;+ let min_import_row = row_range.title_row.unwrap_or(row_range.data_start_row);let sheet = zip.by_name(&sheet_path).map_err(|error| error.to_string())?;let uncompressed_sheet_bytes = sheet.size().max(1);let mut reader = XmlReader::from_reader(BufReader::new(sheet));@@ -4210,7 +4250,9 @@ fn stream_xlsx_rows_to_channel_with_control(.unwrap_or_else(|| (current_row.max(1), current_column.saturating_add(1).max(1)));current_row = position.0;current_column = position.1;- rows.push_cell(position.0, position.1, serde_json::Value::Null, false, progress)?;+ if current_row >= min_import_row && !rows.selected_range_finished(current_row) {+ rows.push_cell(position.0, position.1, serde_json::Value::Null, false, progress)?;+ }}Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {let position = xml_attr_value(&reader, &element, b"r")@@ -4256,16 +4298,18 @@ fn stream_xlsx_rows_to_channel_with_control(}Ok(Event::End(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {if let Some((row, column)) = current_position.take() {- let format_as_text = rows.is_text_source_column(row, column, &text_source_columns);- let value = xlsx_stream_cell_value(- ¤t_cell,- &mut shared_strings,- &styles,- date_1904,- format_as_text,- empty_string_as_null,- )?;- rows.push_cell(row, column, value, current_cell.has_content(), progress)?;+ if row >= min_import_row && !rows.selected_range_finished(row) {+ let format_as_text = rows.is_text_source_column(row, column, &text_source_columns);+ let value = xlsx_stream_cell_value(+ ¤t_cell,+ &mut shared_strings,+ &styles,+ date_1904,+ format_as_text,+ empty_string_as_null,+ )?;+ rows.push_cell(row, column, value, current_cell.has_content(), progress)?;+ }current_cell = XlsxPreviewRawCell::default();}inline_phonetic_depth = 0;@@ -9046,6 +9090,103 @@ mod tests {assert!(state.selected_range_finished(11));}+ #[test]+ fn xlsx_stream_custom_title_row_with_leading_row_cells() {+ // Reproduces #11525: Row 1 has cells starting at column B (e.g. B1 banner),+ // title_row is 2 (starting at column A: A2, B2), and data_start_row is 3 (A3, B3).+ let path = std::env::temp_dir().join(format!("dbx-excel-leading-row-cells-{}.xlsx", uuid::Uuid::new_v4()));+ std::fs::write(+ &path,+ build_styled_test_xlsx(+ false,+ &[+ ("B1", 0, 999.0),+ ("A2", 0, 10.0),+ ("B2", 0, 20.0),+ ("A3", 0, 1.0),+ ("B3", 0, 2.0),+ ("A4", 0, 3.0),+ ("B4", 0, 4.0),+ ],+ ),+ )+ .unwrap();+ let options = TableImportParseOptions {+ title_row: Some(2),+ data_start_row: Some(3),+ ..TableImportParseOptions::default()+ };+ let (preview, _) = parse_xlsx_preview_file_with_options(&path.to_string_lossy(), &options, 50).unwrap();+ assert_eq!(preview.columns, vec!["10", "20"]);+ assert_eq!(+ preview.rows,+ vec![vec![serde_json::json!(1), serde_json::json!(2)], vec![serde_json::json!(3), serde_json::json!(4)],]+ );++ let (sender, mut receiver) = tokio::sync::mpsc::channel(16);+ let stream_result = stream_xlsx_rows_to_channel(+ &path.to_string_lossy(),+ &options,+ 500,+ Some(preview.columns.clone()),+ HashSet::new(),+ false,+ sender,+ );+ assert!(stream_result.is_ok(), "stream_xlsx_rows_to_channel failed: {:?}", stream_result.err());++ let mut streamed_header = None;+ let mut streamed_rows = Vec::new();+ while let Some(message) = receiver.blocking_recv() {+ match message.unwrap() {+ XlsxStreamMessage::Header(header) => streamed_header = Some(header),+ XlsxStreamMessage::Rows(rows) => streamed_rows.extend(rows),+ _ => {}+ }+ }+ assert_eq!(streamed_header, Some(preview.columns));+ assert_eq!(streamed_rows, preview.rows);++ let _ = std::fs::remove_file(path);+ }++ #[test]+ fn xlsx_stream_title_row_with_empty_leading_cell() {+ // Title row 1 only has cell B1; data row 2 has cells A2 and B2.+ let path = std::env::temp_dir().join(format!("dbx-excel-empty-leading-cell-{}.xlsx", uuid::Uuid::new_v4()));+ std::fs::write(&path, build_styled_test_xlsx(false, &[("B1", 0, 20.0), ("A2", 0, 1.0), ("B2", 0, 2.0)]))+ .unwrap();+ let options = TableImportParseOptions {+ title_row: Some(1),+ data_start_row: Some(2),+ ..TableImportParseOptions::default()+ };+ let (preview, _) = parse_xlsx_preview_file_with_options(&path.to_string_lossy(), &options, 50).unwrap();+ assert_eq!(preview.columns, vec!["column_1", "20"]);++ let (sender, mut receiver) = tokio::sync::mpsc::channel(16);+ let stream_result = stream_xlsx_rows_to_channel(+ &path.to_string_lossy(),+ &options,+ 500,+ Some(preview.columns.clone()),+ HashSet::new(),+ false,+ sender,+ );+ assert!(stream_result.is_ok());++ let mut streamed_rows = Vec::new();+ while let Some(message) = receiver.blocking_recv() {+ if let XlsxStreamMessage::Rows(rows) = message.unwrap() {+ streamed_rows.extend(rows);+ }+ }+ assert_eq!(streamed_rows, vec![vec![serde_json::json!(1), serde_json::json!(2)]]);++ let _ = std::fs::remove_file(path);+ }+fn assert_xlsx_empty_string_option(options: TableImportParseOptions, expected_row: Vec<serde_json::Value>) {let path =std::env::temp_dir().join(format!("dbx-table-import-empty-string-option-{}.xlsx", uuid::Uuid::new_v4()));