Apache DataFusion SQL Query Engine
fix(physical-plan): CoalescePartitionsExec panic on wasm32-unknown-unknown (#24890)
Kent Wu0bef032a4a42
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 7438f1c0ca..9e3811e0ad 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -21,6 +21,8 @@use std::sync::Arc;use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};+#[cfg(target_arch = "wasm32")]+use super::stream::RecordBatchStreamAdapter;use super::stream::{ObservedStream, RecordBatchReceiverStream};use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,@@ -229,6 +231,26 @@ impl ExecutionPlan for CoalescePartitionsExec {}_ => {let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);++ // wasm has no tokio reactor, so avoid `JoinSet::spawn`; poll+ // input partitions cooperatively via `select_all` instead.+ #[cfg(target_arch = "wasm32")]+ {+ let elapsed_compute = baseline_metrics.elapsed_compute().clone();+ let _timer = elapsed_compute.timer();++ let mut streams = Vec::with_capacity(input_partitions);+ for i in 0..input_partitions {+ streams.push(self.input.execute(i, Arc::clone(&context))?);+ }+ let merged = futures::stream::select_all(streams);+ return Ok(Box::pin(ObservedStream::new(+ Box::pin(RecordBatchStreamAdapter::new(self.schema(), merged)),+ baseline_metrics,+ self.fetch,+ )));+ }+// record the (very) minimal work done so that// elapsed_compute is not reported as 0let elapsed_compute = baseline_metrics.elapsed_compute().clone();diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rsindex d8da5d4b4f..8e9a7160cd 100644--- a/datafusion/wasmtest/src/lib.rs+++ b/datafusion/wasmtest/src/lib.rs@@ -90,7 +90,9 @@ mod test {execution::context::SessionContext,prelude::CsvReadOptions,};- use datafusion_common::{DataFusionError, test_util::batches_to_string};+ use datafusion_common::{+ DataFusionError, assert_batches_sorted_eq, test_util::batches_to_string,+ };use datafusion_execution::{config::SessionConfig,disk_manager::{DiskManagerBuilder, DiskManagerMode},@@ -234,6 +236,24 @@ mod test {);}+ #[wasm_bindgen_test(unsupported = tokio::test)]+ async fn test_union_all() {+ // Regression: UNION ALL used to panic on wasm via `JoinSet::spawn`.+ let ctx = get_ctx();+ let result = ctx+ .sql("SELECT 1 AS n UNION ALL SELECT 2 AS n")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();++ // CoalescePartitionsExec makes no output-order guarantee; sort+ // before comparing so the assertion tolerates any valid interleaving.+ let expected = ["+---+", "| n |", "+---+", "| 1 |", "| 2 |", "+---+"];+ assert_batches_sorted_eq!(expected, &result);+ }+#[wasm_bindgen_test(unsupported = tokio::test)]async fn test_parquet_write() {let (schema, batch) = create_test_data();