Skip to content

Apache DataFusion SQL Query Engine

fix(physical-plan): CoalescePartitionsExec panic on wasm32-unknown-unknown (#24890)

Kent Wu0bef032a4a42
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 7438f1c0ca..9e3811e0ad 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -21,6 +21,8 @@
use std::sync::Arc;
use super::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
+#[cfg(target_arch = "wasm32")]
+use super::stream::RecordBatchStreamAdapter;
use super::stream::{ObservedStream, RecordBatchReceiverStream};
use super::{
DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,
@@ -229,6 +231,26 @@ impl ExecutionPlan for CoalescePartitionsExec {
}
_ => {
let baseline_metrics = BaselineMetrics::new(&self.metrics, partition);
+
+ // wasm has no tokio reactor, so avoid `JoinSet::spawn`; poll
+ // input partitions cooperatively via `select_all` instead.
+ #[cfg(target_arch = "wasm32")]
+ {
+ let elapsed_compute = baseline_metrics.elapsed_compute().clone();
+ let _timer = elapsed_compute.timer();
+
+ let mut streams = Vec::with_capacity(input_partitions);
+ for i in 0..input_partitions {
+ streams.push(self.input.execute(i, Arc::clone(&context))?);
+ }
+ let merged = futures::stream::select_all(streams);
+ return Ok(Box::pin(ObservedStream::new(
+ Box::pin(RecordBatchStreamAdapter::new(self.schema(), merged)),
+ baseline_metrics,
+ self.fetch,
+ )));
+ }
+
// record the (very) minimal work done so that
// elapsed_compute is not reported as 0
let elapsed_compute = baseline_metrics.elapsed_compute().clone();
diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs
index d8da5d4b4f..8e9a7160cd 100644
--- a/datafusion/wasmtest/src/lib.rs
+++ b/datafusion/wasmtest/src/lib.rs
@@ -90,7 +90,9 @@ mod test {
execution::context::SessionContext,
prelude::CsvReadOptions,
};
- use datafusion_common::{DataFusionError, test_util::batches_to_string};
+ use datafusion_common::{
+ DataFusionError, assert_batches_sorted_eq, test_util::batches_to_string,
+ };
use datafusion_execution::{
config::SessionConfig,
disk_manager::{DiskManagerBuilder, DiskManagerMode},
@@ -234,6 +236,24 @@ mod test {
);
}
+ #[wasm_bindgen_test(unsupported = tokio::test)]
+ async fn test_union_all() {
+ // Regression: UNION ALL used to panic on wasm via `JoinSet::spawn`.
+ let ctx = get_ctx();
+ let result = ctx
+ .sql("SELECT 1 AS n UNION ALL SELECT 2 AS n")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+
+ // CoalescePartitionsExec makes no output-order guarantee; sort
+ // before comparing so the assertion tolerates any valid interleaving.
+ let expected = ["+---+", "| n |", "+---+", "| 1 |", "| 2 |", "+---+"];
+ assert_batches_sorted_eq!(expected, &result);
+ }
+
#[wasm_bindgen_test(unsupported = tokio::test)]
async fn test_parquet_write() {
let (schema, batch) = create_test_data();