Apache DataFusion SQL Query Engine
fix: apply MemTable DELETE and UPDATE at execution, not planning (#25040)
mckee8344cb1e6c
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 1cc7287c32..9cfa32e0d0 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -24,30 +24,34 @@ use std::sync::Arc;use crate::TableProvider;-use arrow::array::{- Array, ArrayRef, BooleanArray, RecordBatch as ArrowRecordBatch, UInt64Array,-};+use arrow::array::{Array, ArrayRef, BooleanArray, UInt64Array};use arrow::compute::kernels::zip::zip;use arrow::compute::{and, filter_record_batch};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::error::Result;use datafusion_common::tree_node::TreeNodeRecursion;-use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};+use datafusion_common::{+ Constraints, DFSchema, SchemaExt, internal_datafusion_err, internal_err,+ not_impl_err, plan_err,+};use datafusion_datasource::memory::{MemSink, MemorySourceConfig};use datafusion_datasource::sink::DataSinkExec;use datafusion_datasource::source::DataSourceExec;+use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_expr::dml::InsertOp;use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, SortExpr, TableType};use datafusion_physical_expr::{- LexOrdering, create_physical_expr, create_physical_sort_exprs,+ EquivalenceProperties, LexOrdering, create_physical_expr, create_physical_sort_exprs,};+use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,- PhysicalExpr, PlanProperties, ReplaceChildrenOptions, collect_partitioned,+ PhysicalExpr, PlanProperties, ReplaceChildrenOptions, apply_expression_roots,+ collect_partitioned,};use datafusion_session::Session;@@ -251,7 +255,8 @@ impl TableProvider for MemTable {'life1: 'async_trait,Self: 'async_trait,{- self.delete_from_boxed(state, filters)+ // Planning a `DELETE` needs no await, so the future is ready at once.+ Box::pin(ready(self.delete_from_inner(state, &filters)))}// Hand-written `#[async_trait]` expansion to reduce compile time. See@@ -267,7 +272,8 @@ impl TableProvider for MemTable {'life1: 'async_trait,Self: 'async_trait,{- self.update_boxed(state, assignments, filters)+ // Planning an `UPDATE` needs no await, so the future is ready at once.+ Box::pin(ready(self.update_inner(state, &assignments, &filters)))}}@@ -357,299 +363,403 @@ impl MemTable {Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None)))}- fn delete_from_boxed<'a>(- &'a self,- state: &'a dyn Session,- filters: Vec<Expr>,- ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {- Box::pin(self.delete_from_inner(state, filters))- }-- async fn delete_from_inner(+ /// Plan a `DELETE`. The rows change when the returned plan runs, not here,+ /// so `EXPLAIN DELETE` prints the plan and the table keeps its rows.+ fn delete_from_inner(&self,state: &dyn Session,- filters: Vec<Expr>,+ filters: &[Expr],) -> Result<Arc<dyn ExecutionPlan>> {- // Early exit if table has no partitions- if self.batches.is_empty() {- return Ok(Arc::new(DmlResultExec::new(0)));- }-- *self.sort_order.lock() = vec![];-- let mut total_deleted: u64 = 0;let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;+ let predicates = create_predicates(filters, &df_schema, state)?;- for partition_data in &self.batches {- let mut partition = partition_data.write().await;- let mut new_batches = Vec::with_capacity(partition.len());-- for batch in partition.iter() {- if batch.num_rows() == 0 {- continue;- }-- // Evaluate filters - None means "match all rows"- let filter_mask = evaluate_filters_to_mask(- &filters,- batch,- &df_schema,- state.execution_props(),- )?;-- let (delete_count, keep_mask) = match filter_mask {- Some(mask) => {- // Count rows where mask is true (will be deleted)- let count = mask.iter().filter(|v| v == &Some(true)).count();- // Keep rows where predicate is false or NULL (SQL three-valued logic)- let keep: BooleanArray =- mask.iter().map(|v| Some(v != Some(true))).collect();- (count, keep)- }- None => {- // No filters = delete all rows- (- batch.num_rows(),- BooleanArray::from(vec![false; batch.num_rows()]),- )- }- };-- total_deleted += delete_count as u64;-- let filtered_batch = filter_record_batch(batch, &keep_mask)?;- if filtered_batch.num_rows() > 0 {- new_batches.push(filtered_batch);- }- }-- *partition = new_batches;- }-- Ok(Arc::new(DmlResultExec::new(total_deleted)))- }-- fn update_boxed<'a>(- &'a self,- state: &'a dyn Session,- assignments: Vec<(String, Expr)>,- filters: Vec<Expr>,- ) -> BoxFuture<'a, Result<Arc<dyn ExecutionPlan>>> {- Box::pin(self.update_inner(state, assignments, filters))+ Ok(Arc::new(MemDeleteExec::new(+ self.batches.clone(),+ Arc::clone(&self.sort_order),+ predicates,+ )))}- async fn update_inner(+ /// Plan an `UPDATE`. The rows change when the returned plan runs, not here,+ /// so `EXPLAIN UPDATE` prints the plan and the table keeps its rows.+ fn update_inner(&self,state: &dyn Session,- assignments: Vec<(String, Expr)>,- filters: Vec<Expr>,+ assignments: &[(String, Expr)],+ filters: &[Expr],) -> Result<Arc<dyn ExecutionPlan>> {- // Early exit if table has no partitions- if self.batches.is_empty() {- return Ok(Arc::new(DmlResultExec::new(0)));- }+ let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;- // Validate column names upfront with clear error messages- let available_columns: Vec<&str> = self- .schema- .fields()- .iter()- .map(|f| f.name().as_str())- .collect();- for (column_name, _) in &assignments {- if self.schema.field_with_name(column_name).is_err() {+ // One entry for each field of the table, in field order. A `Some` entry+ // holds the expression of the `SET` clause for that field.+ let mut set_exprs: Vec<Option<Arc<dyn PhysicalExpr>>> =+ vec![None; self.schema.fields().len()];+ for (column_name, expr) in assignments {+ let Ok(index) = self.schema.index_of(column_name) else {+ let available_columns: Vec<&str> = self+ .schema+ .fields()+ .iter()+ .map(|f| f.name().as_str())+ .collect();return plan_err!("UPDATE failed: column '{}' does not exist. Available columns: {}",column_name,available_columns.join(", "));- }+ };+ set_exprs[index] = Some(create_physical_expr(+ expr,+ &df_schema,+ state.execution_props(),+ &PhysicalPlanningContext::default(),+ )?);}- let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;-- // Create physical expressions for assignments upfront (outside batch loop)- let physical_assignments: HashMap<String, Arc<dyn PhysicalExpr>> = assignments- .iter()- .map(|(name, expr)| {- let physical_expr = create_physical_expr(- expr,- &df_schema,- state.execution_props(),- &PhysicalPlanningContext::default(),- )?;- Ok((name.clone(), physical_expr))- })- .collect::<Result<_>>()?;+ let predicates = create_predicates(filters, &df_schema, state)?;- *self.sort_order.lock() = vec![];+ Ok(Arc::new(MemUpdateExec::new(+ self.batches.clone(),+ Arc::clone(&self.sort_order),+ Arc::clone(&self.schema),+ set_exprs,+ predicates,+ )))+ }+}- let mut total_updated: u64 = 0;+/// Build one physical predicate for each expression of the `WHERE` clause.+///+/// The planner calls this, so a predicate that cannot be planned raises its+/// error while the plan is built and `EXPLAIN` reports it.+fn create_predicates(+ filters: &[Expr],+ df_schema: &DFSchema,+ state: &dyn Session,+) -> Result<Vec<Arc<dyn PhysicalExpr>>> {+ filters+ .iter()+ .map(|filter| {+ create_physical_expr(+ filter,+ df_schema,+ state.execution_props(),+ &PhysicalPlanningContext::default(),+ )+ })+ .collect()+}- for partition_data in &self.batches {- let mut partition = partition_data.write().await;- let mut new_batches = Vec::with_capacity(partition.len());+/// Combine the predicates into one mask over the rows of `batch`. The mask is+/// true for a row that every predicate matches. `None` means there is no+/// `WHERE` clause, which matches every row.+fn evaluate_predicates(+ predicates: &[Arc<dyn PhysicalExpr>],+ batch: &RecordBatch,+) -> Result<Option<BooleanArray>> {+ let mut combined_mask: Option<BooleanArray> = None;- for batch in partition.iter() {- if batch.num_rows() == 0 {- continue;- }+ for predicate in predicates {+ let result = predicate.evaluate(batch)?;+ let array = result.into_array(batch.num_rows())?;+ let bool_array = array+ .as_any()+ .downcast_ref::<BooleanArray>()+ .ok_or_else(|| {+ internal_datafusion_err!("Filter did not evaluate to boolean")+ })?+ .clone();- // Evaluate filters - None means "match all rows"- let filter_mask = evaluate_filters_to_mask(- &filters,- batch,- &df_schema,- state.execution_props(),- )?;+ combined_mask = Some(match combined_mask {+ Some(existing) => and(&existing, &bool_array)?,+ None => bool_array,+ });+ }- let (update_count, update_mask) = match filter_mask {- Some(mask) => {- // Count rows where mask is true (will be updated)- let count = mask.iter().filter(|v| v == &Some(true)).count();- // Normalize mask: only true (not NULL) triggers update- let normalized: BooleanArray =- mask.iter().map(|v| Some(v == Some(true))).collect();- (count, normalized)- }- None => {- // No filters = update all rows- (- batch.num_rows(),- BooleanArray::from(vec![true; batch.num_rows()]),- )- }- };+ Ok(combined_mask)+}- total_updated += update_count as u64;+/// Schema of the single `count` column that a DML plan emits.+fn dml_count_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![Field::new(+ "count",+ DataType::UInt64,+ false,+ )]))+}- if update_count == 0 {- new_batches.push(batch.clone());- continue;- }+/// Properties of a DML plan: one partition, one final batch.+fn dml_plan_properties(schema: &SchemaRef) -> Arc<PlanProperties> {+ Arc::new(PlanProperties::new(+ EquivalenceProperties::new(Arc::clone(schema)),+ Partitioning::UnknownPartitioning(1),+ EmissionType::Final,+ Boundedness::Bounded,+ ))+}- let mut new_columns: Vec<ArrayRef> =- Vec::with_capacity(batch.num_columns());-- for field in self.schema.fields() {- let column_name = field.name();- let original_column =- batch.column_by_name(column_name).ok_or_else(|| {- datafusion_common::DataFusionError::Internal(format!(- "Column '{column_name}' not found in batch"- ))- })?;-- let new_column = if let Some(physical_expr) =- physical_assignments.get(column_name.as_str())- {- // Use evaluate_selection to only evaluate on matching rows.- // This avoids errors (e.g., divide-by-zero) on rows that won't- // be updated. The result is scattered back with nulls for- // non-matching rows, which zip() will replace with originals.- let new_values =- physical_expr.evaluate_selection(batch, &update_mask)?;- let new_array = new_values.into_array(batch.num_rows())?;+/// A single row holding the number of rows the statement changed.+fn count_batch(schema: SchemaRef, rows_affected: u64) -> Result<RecordBatch> {+ let count_array = UInt64Array::from(vec![rows_affected]);+ Ok(RecordBatch::try_new(+ schema,+ vec![Arc::new(count_array) as ArrayRef],+ )?)+}- // Convert to &dyn Array which implements Datum- let new_arr: &dyn Array = new_array.as_ref();- let orig_arr: &dyn Array = original_column.as_ref();- zip(&update_mask, &new_arr, &orig_arr)?- } else {- Arc::clone(original_column)- };+/// Render the predicates as a comma separated list.+fn format_predicates(predicates: &[Arc<dyn PhysicalExpr>]) -> String {+ predicates+ .iter()+ .map(|predicate| predicate.to_string())+ .collect::<Vec<_>>()+ .join(", ")+}- new_columns.push(new_column);- }+/// Deletes the matching rows of a [`MemTable`] when it runs, and emits the count.+///+/// The provider hook that builds this node changes no row, so `EXPLAIN DELETE`+/// prints the plan and the table keeps its rows. Each run of the plan applies+/// the delete once, as [`DataSinkExec`] does for an `INSERT`.+#[derive(Debug)]+struct MemDeleteExec {+ /// Partitions of the target table, shared with the [`MemTable`].+ partitions: Vec<PartitionData>,+ /// Declared sort order of the target table. A delete clears it.+ sort_order: Arc<Mutex<Vec<Vec<SortExpr>>>>,+ /// Predicates of the `WHERE` clause. An empty list matches every row.+ predicates: Vec<Arc<dyn PhysicalExpr>>,+ /// Single `count` column of the output.+ schema: SchemaRef,+ properties: Arc<PlanProperties>,+}- let updated_batch =- ArrowRecordBatch::try_new(Arc::clone(&self.schema), new_columns)?;- new_batches.push(updated_batch);- }+impl MemDeleteExec {+ fn new(+ partitions: Vec<PartitionData>,+ sort_order: Arc<Mutex<Vec<Vec<SortExpr>>>>,+ predicates: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Self {+ let schema = dml_count_schema();+ let properties = dml_plan_properties(&schema);- *partition = new_batches;+ Self {+ partitions,+ sort_order,+ predicates,+ schema,+ properties,}+ }+}- Ok(Arc::new(DmlResultExec::new(total_updated)))+impl DisplayAs for MemDeleteExec {+ fn fmt_as(+ &self,+ t: DisplayFormatType,+ f: &mut std::fmt::Formatter,+ ) -> std::fmt::Result {+ match t {+ DisplayFormatType::Default+ | DisplayFormatType::Verbose+ | DisplayFormatType::TreeRender => {+ write!(f, "MemDeleteExec")?;+ if !self.predicates.is_empty() {+ write!(f, ": predicate=[{}]", format_predicates(&self.predicates))?;+ }+ Ok(())+ }+ }}}-/// Evaluate filter expressions against a batch and return a combined boolean mask.-/// Returns None if filters is empty (meaning "match all rows").-/// The returned mask has true for rows that match the filter predicates.-fn evaluate_filters_to_mask(- filters: &[Expr],- batch: &RecordBatch,- df_schema: &DFSchema,- execution_props: &datafusion_expr::execution_props::ExecutionProps,-) -> Result<Option<BooleanArray>> {- if filters.is_empty() {- return Ok(None);+impl ExecutionPlan for MemDeleteExec {+ fn name(&self) -> &str {+ "MemDeleteExec"}- let mut combined_mask: Option<BooleanArray> = None;+ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }- for filter_expr in filters {- let physical_expr = create_physical_expr(- filter_expr,- df_schema,- execution_props,- &PhysicalPlanningContext::default(),- )?;+ fn properties(&self) -> &Arc<PlanProperties> {+ &self.properties+ }- let result = physical_expr.evaluate(batch)?;- let array = result.into_array(batch.num_rows())?;- let bool_array = array- .as_any()- .downcast_ref::<BooleanArray>()- .ok_or_else(|| {- datafusion_common::DataFusionError::Internal(- "Filter did not evaluate to boolean".to_string(),- )- })?- .clone();+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![]+ }- combined_mask = Some(match combined_mask {- Some(existing) => and(&existing, &bool_array)?,- None => bool_array,+ fn replace_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(self)+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }++ fn execute(+ &self,+ partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ if partition != 0 {+ return internal_err!(+ "MemDeleteExec has one partition, but partition {partition} was requested"+ );+ }++ let partitions = self.partitions.clone();+ let sort_order = Arc::clone(&self.sort_order);+ let predicates = self.predicates.clone();+ let schema = self.schema();++ let stream = futures::stream::once(async move {+ let rows_affected =+ delete_rows(&partitions, &sort_order, &predicates).await?;+ count_batch(schema, rows_affected)});++ Ok(Box::pin(RecordBatchStreamAdapter::new(+ self.schema(),+ stream,+ )))}- Ok(combined_mask)+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ apply_expression_roots(&self.predicates, f)+ }+}++/// Delete the matching rows of every partition and return how many it removed.+async fn delete_rows(+ partitions: &[PartitionData],+ sort_order: &Mutex<Vec<Vec<SortExpr>>>,+ predicates: &[Arc<dyn PhysicalExpr>],+) -> Result<u64> {+ // The surviving rows hold no known order.+ *sort_order.lock() = vec![];++ let mut total_deleted: u64 = 0;++ for partition_data in partitions {+ let mut partition = partition_data.write().await;+ let mut new_batches = Vec::with_capacity(partition.len());++ for batch in partition.iter() {+ if batch.num_rows() == 0 {+ continue;+ }++ let (delete_count, keep_mask) = match evaluate_predicates(predicates, batch)?+ {+ Some(mask) => {+ // Count the rows the mask selects, which are the rows to delete.+ let count = mask.iter().filter(|v| v == &Some(true)).count();+ // Keep the rows for which the predicate is false or NULL,+ // which follows SQL three-valued logic.+ let keep: BooleanArray =+ mask.iter().map(|v| Some(v != Some(true))).collect();+ (count, keep)+ }+ None => {+ // No `WHERE` clause deletes every row.+ (+ batch.num_rows(),+ BooleanArray::from(vec![false; batch.num_rows()]),+ )+ }+ };++ total_deleted += delete_count as u64;++ let filtered_batch = filter_record_batch(batch, &keep_mask)?;+ if filtered_batch.num_rows() > 0 {+ new_batches.push(filtered_batch);+ }+ }++ *partition = new_batches;+ }++ Ok(total_deleted)}-/// Returns a single row with the count of affected rows.+/// Updates the matching rows of a [`MemTable`] when it runs, and emits the count.+///+/// The provider hook that builds this node changes no row, so `EXPLAIN UPDATE`+/// prints the plan and the table keeps its rows. Each run of the plan applies+/// the update once, as [`DataSinkExec`] does for an `INSERT`.#[derive(Debug)]-struct DmlResultExec {- rows_affected: u64,+struct MemUpdateExec {+ /// Partitions of the target table, shared with the [`MemTable`].+ partitions: Vec<PartitionData>,+ /// Declared sort order of the target table. An update clears it.+ sort_order: Arc<Mutex<Vec<Vec<SortExpr>>>>,+ /// Schema of the target table.+ table_schema: SchemaRef,+ /// One entry for each field of the target table, in field order. A `Some`+ /// entry holds the expression of the `SET` clause for that field.+ set_exprs: Vec<Option<Arc<dyn PhysicalExpr>>>,+ /// Predicates of the `WHERE` clause. An empty list matches every row.+ predicates: Vec<Arc<dyn PhysicalExpr>>,+ /// Single `count` column of the output.schema: SchemaRef,properties: Arc<PlanProperties>,}-impl DmlResultExec {- fn new(rows_affected: u64) -> Self {- let schema = Arc::new(Schema::new(vec![Field::new(- "count",- DataType::UInt64,- false,- )]));-- let properties = PlanProperties::new(- datafusion_physical_expr::EquivalenceProperties::new(Arc::clone(&schema)),- Partitioning::UnknownPartitioning(1),- datafusion_physical_plan::execution_plan::EmissionType::Final,- datafusion_physical_plan::execution_plan::Boundedness::Bounded,- );+impl MemUpdateExec {+ fn new(+ partitions: Vec<PartitionData>,+ sort_order: Arc<Mutex<Vec<Vec<SortExpr>>>>,+ table_schema: SchemaRef,+ set_exprs: Vec<Option<Arc<dyn PhysicalExpr>>>,+ predicates: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Self {+ let schema = dml_count_schema();+ let properties = dml_plan_properties(&schema);Self {- rows_affected,+ partitions,+ sort_order,+ table_schema,+ set_exprs,+ predicates,schema,- properties: Arc::new(properties),+ properties,}}++ /// Render the `SET` clauses as `column=expression`, in field order.+ fn format_set_exprs(&self) -> String {+ self.table_schema+ .fields()+ .iter()+ .zip(&self.set_exprs)+ .filter_map(|(field, set_expr)| {+ set_expr+ .as_ref()+ .map(|expr| format!("{}={}", field.name(), expr))+ })+ .collect::<Vec<_>>()+ .join(", ")+ }}-impl DisplayAs for DmlResultExec {+impl DisplayAs for MemUpdateExec {fn fmt_as(&self,t: DisplayFormatType,@@ -659,15 +769,19 @@ impl DisplayAs for DmlResultExec {DisplayFormatType::Default| DisplayFormatType::Verbose| DisplayFormatType::TreeRender => {- write!(f, "DmlResultExec: rows_affected={}", self.rows_affected)+ write!(f, "MemUpdateExec: set=[{}]", self.format_set_exprs())?;+ if !self.predicates.is_empty() {+ write!(f, ", predicate=[{}]", format_predicates(&self.predicates))?;+ }+ Ok(())}}}}-impl ExecutionPlan for DmlResultExec {+impl ExecutionPlan for MemUpdateExec {fn name(&self) -> &str {- "DmlResultExec"+ "MemUpdateExec"}fn schema(&self) -> SchemaRef {@@ -702,28 +816,141 @@ impl ExecutionPlan for DmlResultExec {fn execute(&self,- _partition: usize,- _context: Arc<datafusion_execution::TaskContext>,- ) -> Result<datafusion_execution::SendableRecordBatchStream> {- // Create a single batch with the count- let count_array = UInt64Array::from(vec![self.rows_affected]);- let batch = ArrowRecordBatch::try_new(- Arc::clone(&self.schema),- vec![Arc::new(count_array) as ArrayRef],- )?;+ partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ if partition != 0 {+ return internal_err!(+ "MemUpdateExec has one partition, but partition {partition} was requested"+ );+ }++ let partitions = self.partitions.clone();+ let sort_order = Arc::clone(&self.sort_order);+ let table_schema = Arc::clone(&self.table_schema);+ let set_exprs = self.set_exprs.clone();+ let predicates = self.predicates.clone();+ let schema = self.schema();++ let stream = futures::stream::once(async move {+ let rows_affected = update_rows(+ &partitions,+ &sort_order,+ &table_schema,+ &set_exprs,+ &predicates,+ )+ .await?;+ count_batch(schema, rows_affected)+ });- // Create a stream that yields just this one batch- let stream = futures::stream::iter(vec![Ok(batch)]);Ok(Box::pin(RecordBatchStreamAdapter::new(- Arc::clone(&self.schema),+ self.schema(),stream,)))}fn apply_expressions(&self,- _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,) -> Result<TreeNodeRecursion> {- Ok(TreeNodeRecursion::Continue)+ match apply_expression_roots(self.set_exprs.iter().flatten(), f)? {+ TreeNodeRecursion::Stop => Ok(TreeNodeRecursion::Stop),+ TreeNodeRecursion::Continue | TreeNodeRecursion::Jump => {+ apply_expression_roots(&self.predicates, f)+ }+ }}}++/// Apply the `SET` clauses to the matching rows of every partition and return+/// how many rows changed.+async fn update_rows(+ partitions: &[PartitionData],+ sort_order: &Mutex<Vec<Vec<SortExpr>>>,+ table_schema: &SchemaRef,+ set_exprs: &[Option<Arc<dyn PhysicalExpr>>],+ predicates: &[Arc<dyn PhysicalExpr>],+) -> Result<u64> {+ // The new values hold no known order.+ *sort_order.lock() = vec![];++ let mut total_updated: u64 = 0;++ for partition_data in partitions {+ let mut partition = partition_data.write().await;+ let mut new_batches = Vec::with_capacity(partition.len());++ for batch in partition.iter() {+ if batch.num_rows() == 0 {+ continue;+ }++ let (update_count, update_mask) =+ match evaluate_predicates(predicates, batch)? {+ Some(mask) => {+ // Count the rows the mask selects, which are the rows to update.+ let count = mask.iter().filter(|v| v == &Some(true)).count();+ // Only true, never NULL, selects a row for update.+ let normalized: BooleanArray =+ mask.iter().map(|v| Some(v == Some(true))).collect();+ (count, normalized)+ }+ None => {+ // No `WHERE` clause updates every row.+ (+ batch.num_rows(),+ BooleanArray::from(vec![true; batch.num_rows()]),+ )+ }+ };++ total_updated += update_count as u64;++ if update_count == 0 {+ new_batches.push(batch.clone());+ continue;+ }++ let mut new_columns: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());++ for (index, field) in table_schema.fields().iter().enumerate() {+ let column_name = field.name();+ let original_column =+ batch.column_by_name(column_name).ok_or_else(|| {+ internal_datafusion_err!(+ "Column '{column_name}' not found in batch"+ )+ })?;++ let new_column = match set_exprs.get(index).and_then(|e| e.as_ref()) {+ Some(set_expr) => {+ // `evaluate_selection` evaluates the matching rows only,+ // which keeps an error such as a divide by zero away from+ // the rows the statement does not touch. It returns NULL+ // for the other rows, and `zip` puts the originals back.+ let new_values =+ set_expr.evaluate_selection(batch, &update_mask)?;+ let new_array = new_values.into_array(batch.num_rows())?;++ // Convert to &dyn Array, which implements Datum+ let new_arr: &dyn Array = new_array.as_ref();+ let orig_arr: &dyn Array = original_column.as_ref();+ zip(&update_mask, &new_arr, &orig_arr)?+ }+ None => Arc::clone(original_column),+ };++ new_columns.push(new_column);+ }++ let updated_batch =+ RecordBatch::try_new(Arc::clone(table_schema), new_columns)?;+ new_batches.push(updated_batch);+ }++ *partition = new_batches;+ }++ Ok(total_updated)+}diff --git a/datafusion/core/src/datasource/memory_test.rs b/datafusion/core/src/datasource/memory_test.rsindex 033a8036f5..ff464ec3d3 100644--- a/datafusion/core/src/datasource/memory_test.rs+++ b/datafusion/core/src/datasource/memory_test.rs@@ -20,7 +20,7 @@ mod tests {use crate::datasource::MemTable;use crate::datasource::{DefaultTableSource, provider_as_source};- use crate::physical_plan::collect;+ use crate::physical_plan::{ExecutionPlan, collect};use crate::prelude::SessionContext;use arrow::array::{AsArray, Int32Array};use arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt64Type};@@ -28,9 +28,13 @@ mod tests {use arrow::record_batch::RecordBatch;use arrow_schema::SchemaRef;use datafusion_catalog::TableProvider;- use datafusion_common::{Constraint, Constraints, DataFusionError, Result};- use datafusion_expr::LogicalPlanBuilder;+ use datafusion_common::tree_node::TreeNodeRecursion;+ use datafusion_common::{+ Constraint, Constraints, DataFusionError, Result, assert_contains,+ };use datafusion_expr::dml::InsertOp;+ use datafusion_expr::{Expr, LogicalPlanBuilder, col, lit};+ use datafusion_physical_expr::utils::collect_columns;use futures::StreamExt;use std::collections::HashMap;use std::sync::Arc;@@ -627,4 +631,410 @@ mod tests {);Ok(())}++ /// A schema of one non-nullable Int32 column called "a".+ fn one_column_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]))+ }++ /// A batch of the rows 1, 2 and 3 in the column "a".+ fn one_column_batch(schema: &SchemaRef) -> Result<RecordBatch> {+ Ok(RecordBatch::try_new(+ Arc::clone(schema),+ vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],+ )?)+ }++ /// A `MemTable` of one partition holding the rows 1, 2 and 3.+ fn one_column_table() -> Result<MemTable> {+ let schema = one_column_schema();+ MemTable::try_new(Arc::clone(&schema), vec![vec![one_column_batch(&schema)?]])+ }++ /// A `MemTable` that holds no partition. `MemTable::try_new` rejects an+ /// empty partition list, so a caller reaches this state through the public+ /// `batches` field.+ fn zero_partition_table(schema: SchemaRef) -> Result<MemTable> {+ let mut table = MemTable::try_new(schema, vec![vec![]])?;+ table.batches.clear();+ Ok(table)+ }++ /// Run a DELETE or an UPDATE plan and return the count that it emits.+ async fn run_dml(+ plan: Arc<dyn ExecutionPlan>,+ session_ctx: &SessionContext,+ ) -> Result<u64> {+ Ok(extract_count(collect(plan, session_ctx.task_ctx()).await?))+ }++ /// Read one partition of a `MemTable` as a plain vector of batches.+ async fn read_partition(table: &MemTable, partition: usize) -> Vec<RecordBatch> {+ table.batches[partition].read().await.clone()+ }++ /// The values of the first column of `batch`, which must hold no null.+ fn column_values(batch: &RecordBatch) -> Vec<i32> {+ batch+ .column(0)+ .as_primitive::<Int32Type>()+ .iter()+ .map(|value| value.expect("expected non null"))+ .collect()+ }++ // SQL cannot hold an unpolled stream or execute the same physical plan twice.+ #[tokio::test]+ async fn test_dml_execution_lifecycle() -> Result<()> {+ for update in [false, true] {+ let session_ctx = SessionContext::new();+ let schema = one_column_schema();+ let batch = one_column_batch(&schema)?;+ let ordering = vec![vec![col("a").sort(true, false)]];+ let table = MemTable::try_new(schema, vec![vec![batch.clone()], vec![]])?+ .with_sort_order(ordering.clone());+ let filters = vec![col("a").gt(lit(1))];+ let plan = if update {+ table+ .update(+ &session_ctx.state(),+ vec![("a".to_string(), col("a") + lit(10))],+ filters,+ )+ .await?+ } else {+ table.delete_from(&session_ctx.state(), filters).await?+ };++ let stream = plan.execute(0, session_ctx.task_ctx())?;+ assert_eq!(stream.schema(), plan.schema());+ drop(stream);+ assert_eq!(+ column_values(&read_partition(&table, 0).await[0]),+ vec![1, 2, 3]+ );+ assert_eq!(*table.sort_order.lock(), ordering);++ // Rows added after planning must participate, including duplicates+ // in separate batches and an initially empty partition.+ table.batches[1]+ .write()+ .await+ .extend([batch.clone(), batch]);+ assert_eq!(run_dml(Arc::clone(&plan), &session_ctx).await?, 6);+ assert!(table.sort_order.lock().is_empty());+ let expected = if update { vec![1, 12, 13] } else { vec![1] };+ let mut actual = Vec::new();+ for partition in &table.batches {+ for batch in partition.read().await.iter() {+ actual.extend(column_values(batch));+ }+ }+ actual.sort_unstable();+ let mut expected = expected.repeat(3);+ expected.sort_unstable();+ assert_eq!(actual, expected);++ // Reusing the plan reevaluates current rows and emits a fresh count.+ assert_eq!(+ run_dml(plan, &session_ctx).await?,+ if update { 6 } else { 0 }+ );+ let expected = if update { vec![1, 22, 23] } else { vec![1] };+ let mut actual = Vec::new();+ for partition in &table.batches {+ for batch in partition.read().await.iter() {+ actual.extend(column_values(batch));+ }+ }+ actual.sort_unstable();+ let mut expected = expected.repeat(3);+ expected.sort_unstable();+ assert_eq!(actual, expected);+ }+ Ok(())+ }++ // Expression visitors must see both assignments and predicates, while+ // respecting early termination across those two groups.+ #[tokio::test]+ async fn test_dml_expression_visitors() -> Result<()> {+ let session_ctx = SessionContext::new();+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ Field::new("untouched", DataType::Int32, false),+ ]));+ let table = MemTable::try_new(schema, vec![vec![]])?;+ let state = session_ctx.state();+ let delete = table.delete_from(&state, vec![col("a").gt(lit(1))]).await?;+ let update = table+ .update(+ &state,+ vec![("b".to_string(), col("b") + lit(10))],+ vec![col("a").gt(lit(1))],+ )+ .await?;++ for (plan, expected) in [(delete, vec!["a"]), (update, vec!["a", "b"])] {+ for recursion in [TreeNodeRecursion::Continue, TreeNodeRecursion::Jump] {+ let mut columns = Vec::new();+ plan.apply_expressions(&mut |expr| {+ columns.extend(+ collect_columns(expr)+ .iter()+ .map(|col| col.name().to_string()),+ );+ Ok(recursion)+ })?;+ columns.sort();+ assert_eq!(columns, expected);+ }++ let mut visits = 0;+ let result = plan.apply_expressions(&mut |_| {+ visits += 1;+ Ok(TreeNodeRecursion::Stop)+ })?;+ assert_eq!(result, TreeNodeRecursion::Stop);+ assert_eq!(visits, 1);++ let err = plan+ .apply_expressions(&mut |_| {+ Err(DataFusionError::Execution("visitor failed".to_string()))+ })+ .unwrap_err();+ assert_contains!(err.to_string(), "visitor failed");+ }+ Ok(())+ }++ // A DELETE on a table without a partition affects no row+ #[tokio::test]+ async fn test_delete_from_zero_partition() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = zero_partition_table(one_column_schema())?;++ let plan = table.delete_from(&state, vec![col("a").gt(lit(1))]).await?;+ assert_eq!(run_dml(plan, &session_ctx).await?, 0);+ Ok(())+ }++ // An UPDATE on a table without a partition affects no row+ #[tokio::test]+ async fn test_update_zero_partition() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = zero_partition_table(one_column_schema())?;++ let plan = table+ .update(&state, vec![("a".to_string(), lit(7))], vec![])+ .await?;+ assert_eq!(run_dml(plan, &session_ctx).await?, 0);+ Ok(())+ }++ // A DELETE skips a batch of no row and drops it from the partition+ #[tokio::test]+ async fn test_delete_from_empty_batch() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let schema = one_column_schema();+ let table = MemTable::try_new(+ Arc::clone(&schema),+ vec![vec![+ RecordBatch::new_empty(Arc::clone(&schema)),+ one_column_batch(&schema)?,+ ]],+ )?;++ let plan = table.delete_from(&state, vec![col("a").gt(lit(1))]).await?;+ assert_eq!(run_dml(plan, &session_ctx).await?, 2);++ // The empty batch is gone and the row 1 remains+ let partition = read_partition(&table, 0).await;+ assert_eq!(partition.len(), 1);+ assert_eq!(column_values(&partition[0]), vec![1]);+ Ok(())+ }++ // An UPDATE skips a batch of no row and drops it from the partition+ #[tokio::test]+ async fn test_update_empty_batch() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let schema = one_column_schema();+ let table = MemTable::try_new(+ Arc::clone(&schema),+ vec![vec![+ RecordBatch::new_empty(Arc::clone(&schema)),+ one_column_batch(&schema)?,+ ]],+ )?;++ let plan = table+ .update(+ &state,+ vec![("a".to_string(), lit(7))],+ vec![col("a").gt(lit(1))],+ )+ .await?;+ assert_eq!(run_dml(plan, &session_ctx).await?, 2);++ // The empty batch is gone and the rows 2 and 3 now hold 7+ let partition = read_partition(&table, 0).await;+ assert_eq!(partition.len(), 1);+ assert_eq!(column_values(&partition[0]), vec![1, 7, 7]);+ Ok(())+ }++ // The DELETE plan has one partition and rejects a request for another+ #[tokio::test]+ async fn test_delete_exec_rejects_other_partition() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let plan = table.delete_from(&state, vec![]).await?;+ let Err(err) = plan.execute(1, session_ctx.task_ctx()) else {+ panic!("expected an error for partition 1");+ };+ assert_contains!(+ err.strip_backtrace(),+ "MemDeleteExec has one partition, but partition 1 was requested"+ );++ // The failed request leaves the rows alone+ assert_eq!(read_partition(&table, 0).await[0].num_rows(), 3);+ Ok(())+ }++ // The UPDATE plan has one partition and rejects a request for another+ #[tokio::test]+ async fn test_update_exec_rejects_other_partition() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let plan = table+ .update(&state, vec![("a".to_string(), lit(7))], vec![])+ .await?;+ let Err(err) = plan.execute(1, session_ctx.task_ctx()) else {+ panic!("expected an error for partition 1");+ };+ assert_contains!(+ err.strip_backtrace(),+ "MemUpdateExec has one partition, but partition 1 was requested"+ );++ // The failed request leaves the rows alone+ assert_eq!(+ column_values(&read_partition(&table, 0).await[0]),+ vec![1, 2, 3]+ );+ Ok(())+ }++ // An UPDATE whose `SET` clause names an unknown column fails while the plan+ // is built, so `EXPLAIN` reports it+ #[tokio::test]+ async fn test_update_unknown_set_column() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let err = table+ .update(&state, vec![("nonexistent".to_string(), lit(7))], vec![])+ .await+ .unwrap_err();+ assert_contains!(+ err.strip_backtrace(),+ "UPDATE failed: column 'nonexistent' does not exist. Available columns: a"+ );+ Ok(())+ }++ // A DELETE whose `WHERE` clause is not a predicate fails when the plan runs+ #[tokio::test]+ async fn test_delete_from_non_boolean_filter() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let plan = table.delete_from(&state, vec![lit(1)]).await?;+ let err = run_dml(plan, &session_ctx).await.unwrap_err();+ assert_contains!(err.strip_backtrace(), "Filter did not evaluate to boolean");++ // The failed run leaves the rows alone+ assert_eq!(read_partition(&table, 0).await[0].num_rows(), 3);+ Ok(())+ }++ // An UPDATE whose `WHERE` clause is not a predicate fails when the plan runs+ #[tokio::test]+ async fn test_update_non_boolean_filter() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let plan = table+ .update(&state, vec![("a".to_string(), lit(7))], vec![lit(1)])+ .await?;+ let err = run_dml(plan, &session_ctx).await.unwrap_err();+ assert_contains!(err.strip_backtrace(), "Filter did not evaluate to boolean");+ Ok(())+ }++ // An UPDATE reports the column that a batch of the table does not hold+ #[tokio::test]+ async fn test_update_column_missing_from_batch() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, true),+ ]));+ let table = MemTable::try_new(+ Arc::clone(&table_schema),+ vec![vec![RecordBatch::try_new(+ table_schema,+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3])),+ Arc::new(Int32Array::from(vec![4, 5, 6])),+ ],+ )?]],+ )?;+ // `try_new` rejects a batch that misses a column of the table, so the+ // test writes one straight into the partition. The UPDATE reports the+ // missing column instead of a panic.+ let batch = one_column_batch(&one_column_schema())?;+ *table.batches[0].write().await = vec![batch];++ let plan = table+ .update(&state, vec![("a".to_string(), lit(7))], vec![])+ .await?;+ let err = run_dml(plan, &session_ctx).await.unwrap_err();+ assert_contains!(err.strip_backtrace(), "Column 'b' not found in batch");+ Ok(())+ }++ // An UPDATE reports an assignment of a value of the wrong type+ #[tokio::test]+ async fn test_update_assignment_type_mismatch() -> Result<()> {+ let session_ctx = SessionContext::new();+ let state = session_ctx.state();+ let table = one_column_table()?;++ let assignment: (String, Expr) = ("a".to_string(), lit("seven"));+ let plan = table+ .update(&state, vec![assignment], vec![col("a").gt(lit(1))])+ .await?;+ let err = run_dml(plan, &session_ctx).await.unwrap_err();+ assert_contains!(+ err.strip_backtrace(),+ "arguments need to have the same data type"+ );+ Ok(())+ }}diff --git a/datafusion/sqllogictest/test_files/delete.slt b/datafusion/sqllogictest/test_files/delete.sltindex 9682cc27e4..5c2da467bb 100644--- a/datafusion/sqllogictest/test_files/delete.slt+++ b/datafusion/sqllogictest/test_files/delete.slt@@ -36,7 +36,7 @@ logical_plan02)--TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExec# Filtered by existing columns@@ -49,7 +49,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExec: predicate=[CAST(a@0 AS Int64) = 1, CAST(b@1 AS Int64) = 2, c@2 > CAST(3 AS Float64), CAST(d@3 AS Int64) != 4]# Filtered by existing columns, using qualified and unqualified names@@ -62,7 +62,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExec: predicate=[CAST(a@0 AS Int64) = 1, CAST(b@1 AS Int64) = 2, c@2 > CAST(3 AS Float64), CAST(d@3 AS Int64) != 4]# Filtered by a mix of columns and literal predicates@@ -75,7 +75,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExec: predicate=[CAST(a@0 AS Int64) = 1, 1 = 1, true]# Deleting by columns that do not exist returns an error@@ -126,7 +126,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExecquery TT@@ -139,7 +139,7 @@ logical_plan04)------TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemDeleteExec: predicate=[CAST(a@0 AS Int64) = 1, b@1 = CAST(2 AS Utf8View)]# Config resetstatement okdiff --git a/datafusion/sqllogictest/test_files/dml_delete.slt b/datafusion/sqllogictest/test_files/dml_delete.sltindex 34b5808f96..296baa729f 100644--- a/datafusion/sqllogictest/test_files/dml_delete.slt+++ b/datafusion/sqllogictest/test_files/dml_delete.slt@@ -200,3 +200,50 @@ SELECT * FROM test_delete_in;statement okDROP TABLE test_delete_in;++# Test that EXPLAIN DELETE prints the plan and deletes nothing.+# Regression test for https://github.com/apache/datafusion/issues/24656:+# the delete used to run while the physical plan was built.+statement ok+CREATE TABLE test_delete_explain AS VALUES (1, 'a'), (2, 'b'), (3, 'c');++statement ok+EXPLAIN DELETE FROM test_delete_explain WHERE column1 > 1;++query IT rowsort+SELECT * FROM test_delete_explain;+----+1 a+2 b+3 c++# EXPLAIN ANALYZE runs the plan, so it does delete, and it deletes once.+statement ok+EXPLAIN ANALYZE DELETE FROM test_delete_explain WHERE column1 > 1;++query IT rowsort+SELECT * FROM test_delete_explain;+----+1 a++statement ok+DROP TABLE test_delete_explain;++# Predicate errors belong to execution; EXPLAIN only plans the expression.+statement ok+CREATE TABLE test_delete_error AS VALUES (0), (2);++statement ok+EXPLAIN DELETE FROM test_delete_error WHERE 10 / column1 > 1;++statement error Divide by zero+DELETE FROM test_delete_error WHERE 10 / column1 > 1;++query I rowsort+SELECT * FROM test_delete_error;+----+0+2++statement ok+DROP TABLE test_delete_error;diff --git a/datafusion/sqllogictest/test_files/dml_update.slt b/datafusion/sqllogictest/test_files/dml_update.sltindex 160a8b63a3..d0712afff2 100644--- a/datafusion/sqllogictest/test_files/dml_update.slt+++ b/datafusion/sqllogictest/test_files/dml_update.slt@@ -253,7 +253,15 @@ statement okINSERT INTO test_update_error VALUES (1, 'test');statement error No field named nonexistent-UPDATE test_update_error SET nonexistent = 'value';+EXPLAIN UPDATE test_update_error SET nonexistent = 'value';++statement error No field named nonexistent+EXPLAIN UPDATE test_update_error SET name = 'value' WHERE nonexistent = 1;++query IT+SELECT * FROM test_update_error;+----+1 teststatement okDROP TABLE test_update_error;@@ -282,5 +290,78 @@ SELECT * FROM test_update_div;2 2 503 5 20+# Planning must not evaluate assignments, but execution must report their errors.+statement ok+EXPLAIN UPDATE test_update_div SET result = 100 / divisor;++statement error Divide by zero+UPDATE test_update_div SET result = 100 / divisor;++query III rowsort+SELECT * FROM test_update_div;+----+1 0 0+2 2 50+3 5 20+statement okDROP TABLE test_update_div;++# Test that EXPLAIN UPDATE prints the plan and updates nothing.+# Regression test for https://github.com/apache/datafusion/issues/24656:+# the update used to run while the physical plan was built.+statement ok+CREATE TABLE test_update_explain AS VALUES (1, 'a'), (2, 'b'), (3, 'c');++statement ok+EXPLAIN UPDATE test_update_explain SET column1 = column1 + 10 WHERE column1 > 1;++query IT rowsort+SELECT * FROM test_update_explain;+----+1 a+2 b+3 c++# EXPLAIN ANALYZE runs the plan, so it does update. Each matching value goes+# up by ten once, so a second run of the plan would show 22 and 23.+statement ok+EXPLAIN ANALYZE UPDATE test_update_explain SET column1 = column1 + 10 WHERE column1 > 1;++query IT rowsort+SELECT * FROM test_update_explain;+----+1 a+12 b+13 c++statement ok+DROP TABLE test_update_explain;++# A nullable source can be assigned to a NOT NULL column when selected values+# are non-null. Selecting its NULL must fail at execution.+statement ok+CREATE TABLE test_update_not_null (id INT NOT NULL, value INT);++statement ok+INSERT INTO test_update_not_null VALUES (1, 10), (2, NULL);++query I+UPDATE test_update_not_null SET id = value WHERE value IS NOT NULL;+----+1++statement ok+EXPLAIN UPDATE test_update_not_null SET id = value;++statement error non-nullable but contains null+UPDATE test_update_not_null SET id = value;++query II rowsort+SELECT * FROM test_update_not_null;+----+10 10+2 NULL++statement ok+DROP TABLE test_update_not_null;diff --git a/datafusion/sqllogictest/test_files/update.slt b/datafusion/sqllogictest/test_files/update.sltindex b74872cb29..7ffeb8f39a 100644--- a/datafusion/sqllogictest/test_files/update.slt+++ b/datafusion/sqllogictest/test_files/update.slt@@ -35,7 +35,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemUpdateExec: set=[a=CAST(1 AS Int32), b=CAST(2 AS Utf8View), c=3, d=CAST(NULL AS Int32)]query TTexplain update t1 set a=c+1, b=a, c=c+1.0, d=b;@@ -46,7 +46,7 @@ logical_plan03)----TableScan: t1physical_plan01)CooperativeExec-02)--DmlResultExec: rows_affected=0+02)--MemUpdateExec: set=[a=CAST(c@2 + CAST(1 AS Float64) AS Int32), b=CAST(a@0 AS Utf8View), c=c@2 + 1, d=CAST(b@1 AS Int32)]statement okcreate table t2(a int, b varchar, c double, d int);