Skip to content

Apache DataFusion SQL Query Engine

fix: preserve the error type of a failing parquet row filter predicate (#24638)

Adrian Garcia Badaracco1064661e82ba
diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rs
index dabb2f35b2..25952f8193 100644
--- a/datafusion/core/tests/parquet/filter_pushdown.rs
+++ b/datafusion/core/tests/parquet/filter_pushdown.rs
@@ -26,10 +26,12 @@
//! select * from data limit 10;
//! ```
+use arrow::array::{ArrayRef, Int32Array, StringArray};
use arrow::compute::concat_batches;
+use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
-use datafusion::physical_plan::collect;
use datafusion::physical_plan::metrics::{MetricValue, MetricsSet};
+use datafusion::physical_plan::{collect, displayable};
use datafusion::prelude::{
Expr, ParquetReadOptions, SessionContext, col, lit, lit_timestamp_nano,
};
@@ -37,10 +39,14 @@ use datafusion::test_util::parquet::{ParquetScanOptions, TestParquetFile};
use datafusion_expr::utils::{conjunction, disjunction, split_conjunction};
use std::path::Path;
+use datafusion_common::DataFusionError;
use datafusion_common::test_util::parquet_test_data;
use datafusion_execution::config::SessionConfig;
use itertools::Itertools;
+use parquet::arrow::ArrowWriter;
use parquet::file::properties::WriterProperties;
+use std::fs::File;
+use std::sync::Arc;
use tempfile::TempDir;
/// how many rows of generated data to write to our parquet file (arbitrary)
@@ -746,3 +752,60 @@ impl PredicateCacheTest {
Ok(())
}
}
+
+/// A predicate that is pushed into the parquet decoder and then fails while it
+/// is being evaluated must report the original error, so that callers can still
+/// tell a user error apart from an internal one.
+#[tokio::test]
+async fn pushed_down_predicate_reports_the_original_error() {
+ let tempdir = TempDir::new_in(Path::new(".")).unwrap();
+ let path = tempdir.path().join("cast_error.parquet");
+
+ // `v` is never referenced by the predicate, so the projection always has a
+ // column that only the scan can supply and a narrow-projection pushdown
+ // heuristic has no reason to decline this scan
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "s",
+ Arc::new(StringArray::from(vec!["not_an_int"])) as ArrayRef,
+ ),
+ ("v", Arc::new(Int32Array::from(vec![1])) as ArrayRef),
+ ])
+ .unwrap();
+ let mut writer =
+ ArrowWriter::try_new(File::create(&path).unwrap(), batch.schema(), None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let mut config = SessionConfig::new();
+ config.options_mut().execution.parquet.pushdown_filters = true;
+ let ctx = SessionContext::new_with_config(config);
+ ctx.register_parquet("t", path.to_str().unwrap(), ParquetReadOptions::default())
+ .await
+ .unwrap();
+
+ // Casting the column in the file to `Int32` fails on this data
+ let df = ctx
+ .sql("SELECT * FROM t WHERE CAST(s AS INT) = 1")
+ .await
+ .unwrap();
+
+ let plan = df.clone().create_physical_plan().await.unwrap();
+ let plan = displayable(plan.as_ref()).indent(false).to_string();
+ assert!(
+ !plan.contains("FilterExec"),
+ "the predicate has to reach the decoder for this test to mean anything, \
+ but a FilterExec here means pushdown was declined:\n{plan}"
+ );
+
+ let err = df.collect().await.unwrap_err();
+ let root = err.find_root();
+ assert!(
+ matches!(
+ root,
+ DataFusionError::ArrowError(inner, _)
+ if matches!(inner.as_ref(), ArrowError::CastError(_))
+ ),
+ "expected the original cast error, got {root:?}"
+ );
+}
diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs
index d5e085b637..267f0f5917 100644
--- a/datafusion/datasource-parquet/src/row_filter.rs
+++ b/datafusion/datasource-parquet/src/row_filter.rs
@@ -166,9 +166,12 @@ impl ArrowPredicate for DatafusionArrowPredicate {
timer.stop();
Ok(bool_arr)
})
+ // `ExternalError` is the only `ArrowError` variant that keeps a
+ // source, and therefore the only one that leaves the original error
+ // recoverable (see `DataFusionError::find_root`)
.map_err(|e| {
- ArrowError::ComputeError(format!(
- "Error evaluating filter predicate: {e:?}"
+ ArrowError::ExternalError(Box::new(
+ e.context("Error evaluating filter predicate"),
))
})
}
@@ -545,13 +548,13 @@ pub(crate) fn row_filter_from_prebuilt(
mod test {
use super::*;
use arrow::datatypes::{DataType, Fields};
- use datafusion_common::ScalarValue;
+ use datafusion_common::{DataFusionError, ScalarValue};
use arrow::array::{
Int32Array, ListBuilder, StringArray, StringBuilder, StructArray,
};
use arrow::datatypes::{Field, TimeUnit::Nanosecond};
- use datafusion_expr::{Expr, col};
+ use datafusion_expr::{Cast, Expr, col, lit};
use datafusion_functions::core::get_field;
use datafusion_functions_nested::array_has::{
array_has_all_udf, array_has_any_udf, array_has_udf,
@@ -692,6 +695,82 @@ mod test {
assert!(matches!(filtered, Ok(a) if a == BooleanArray::from(vec![true; 8])));
}
+ /// A predicate that fails while it is being evaluated must report the
+ /// original error, not an opaque string, so that callers can still tell a
+ /// user error apart from an internal one.
+ #[test]
+ fn evaluate_reports_the_original_error() {
+ let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(StringArray::from(vec!["not_an_int"]))],
+ )
+ .expect("record batch");
+
+ let file = NamedTempFile::new().expect("temp file");
+ let mut writer =
+ ArrowWriter::try_new(file.reopen().unwrap(), Arc::clone(&schema), None)
+ .expect("writer");
+ writer.write(&batch).expect("write batch");
+ writer.close().expect("close writer");
+
+ let parquet_reader_builder =
+ ParquetRecordBatchReaderBuilder::try_new(file.reopen().expect("reopen file"))
+ .expect("reader builder");
+ let metadata = parquet_reader_builder.metadata().clone();
+ let file_schema = parquet_reader_builder.schema().clone();
+
+ // Casting the column in the file to `Int32` fails on this data
+ let expr = Expr::Cast(Cast::new(Box::new(col("s")), DataType::Int32)).eq(lit(1));
+ let expr = logical2physical(&expr, &file_schema);
+ let candidate = FilterCandidateBuilder::new(expr, Arc::clone(&file_schema))
+ .build(&metadata)
+ .expect("building candidate")
+ .expect("candidate expected");
+
+ let mut predicate = DatafusionArrowPredicate::try_new(
+ candidate,
+ Count::new(),
+ Count::new(),
+ Time::new(),
+ )
+ .expect("creating filter predicate");
+
+ let mut parquet_reader = parquet_reader_builder
+ .with_projection(predicate.projection().clone())
+ .build()
+ .expect("building reader");
+ let first_rb = parquet_reader
+ .next()
+ .expect("expected record batch")
+ .expect("expected error free record batch");
+
+ let err = predicate
+ .evaluate(first_rb)
+ .expect_err("evaluating the predicate should fail");
+
+ // The cast failure is still reachable, rather than being flattened into
+ // an untyped `ArrowError::ComputeError`
+ let err = DataFusionError::from(err);
+ let root = err.find_root();
+ assert!(
+ matches!(
+ root,
+ DataFusionError::ArrowError(inner, _)
+ if matches!(inner.as_ref(), ArrowError::CastError(_))
+ ),
+ "expected the original cast error, got {root:?}"
+ );
+
+ // and the message still says where the failure happened
+ let message = err.to_string();
+ assert!(
+ message.contains("Error evaluating filter predicate"),
+ "{message}"
+ );
+ assert!(message.contains("Cannot cast string"), "{message}");
+ }
+
#[test]
fn struct_data_structures_prevent_pushdown() {
let table_schema = Arc::new(Schema::new(vec![Field::new(