Apache DataFusion SQL Query Engine
feat: Add DELETE/UPDATE hooks to TableProvider trait and to MemTable implementation (#19142)
Ethan Urbanskid13d89129b77
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 47f773fe9b..7865eb016b 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -24,7 +24,12 @@ use std::sync::Arc;use crate::TableProvider;-use arrow::datatypes::SchemaRef;+use arrow::array::{+ Array, ArrayRef, BooleanArray, RecordBatch as ArrowRecordBatch, UInt64Array,+};+use arrow::compute::kernels::zip::zip;+use arrow::compute::{and, filter_record_batch};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::error::Result;use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};@@ -34,10 +39,14 @@ use datafusion_datasource::sink::DataSinkExec;use datafusion_datasource::source::DataSourceExec;use datafusion_expr::dml::InsertOp;use datafusion_expr::{Expr, SortExpr, TableType};-use datafusion_physical_expr::{LexOrdering, create_physical_sort_exprs};+use datafusion_physical_expr::{+ LexOrdering, create_physical_expr, create_physical_sort_exprs,+};use datafusion_physical_plan::repartition::RepartitionExec;+use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{- ExecutionPlan, ExecutionPlanProperties, Partitioning, common,+ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,+ PlanProperties, common,};use datafusion_session::Session;@@ -295,4 +304,338 @@ impl TableProvider for MemTable {fn get_column_default(&self, column: &str) -> Option<&Expr> {self.column_defaults.get(column)}++ async fn delete_from(+ &self,+ state: &dyn Session,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Early exit if table has no partitions+ if self.batches.is_empty() {+ return Ok(Arc::new(DmlResultExec::new(0)));+ }++ *self.sort_order.lock() = vec![];++ let mut total_deleted: u64 = 0;+ let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;++ for partition_data in &self.batches {+ let mut partition = partition_data.write().await;+ let mut new_batches = Vec::with_capacity(partition.len());++ for batch in partition.iter() {+ if batch.num_rows() == 0 {+ continue;+ }++ // Evaluate filters - None means "match all rows"+ let filter_mask = evaluate_filters_to_mask(+ &filters,+ batch,+ &df_schema,+ state.execution_props(),+ )?;++ let (delete_count, keep_mask) = match filter_mask {+ Some(mask) => {+ // Count rows where mask is true (will be deleted)+ let count = mask.iter().filter(|v| v == &Some(true)).count();+ // Keep rows where predicate is false or NULL (SQL three-valued logic)+ let keep: BooleanArray =+ mask.iter().map(|v| Some(v != Some(true))).collect();+ (count, keep)+ }+ None => {+ // No filters = delete all rows+ (+ batch.num_rows(),+ BooleanArray::from(vec![false; batch.num_rows()]),+ )+ }+ };++ total_deleted += delete_count as u64;++ let filtered_batch = filter_record_batch(batch, &keep_mask)?;+ if filtered_batch.num_rows() > 0 {+ new_batches.push(filtered_batch);+ }+ }++ *partition = new_batches;+ }++ Ok(Arc::new(DmlResultExec::new(total_deleted)))+ }++ async fn update(+ &self,+ state: &dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Early exit if table has no partitions+ if self.batches.is_empty() {+ return Ok(Arc::new(DmlResultExec::new(0)));+ }++ // Validate column names upfront with clear error messages+ let available_columns: Vec<&str> = self+ .schema+ .fields()+ .iter()+ .map(|f| f.name().as_str())+ .collect();+ for (column_name, _) in &assignments {+ if self.schema.field_with_name(column_name).is_err() {+ return plan_err!(+ "UPDATE failed: column '{}' does not exist. Available columns: {}",+ column_name,+ available_columns.join(", ")+ );+ }+ }++ let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;++ // Create physical expressions for assignments upfront (outside batch loop)+ let physical_assignments: HashMap<+ String,+ Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ > = assignments+ .iter()+ .map(|(name, expr)| {+ let physical_expr =+ create_physical_expr(expr, &df_schema, state.execution_props())?;+ Ok((name.clone(), physical_expr))+ })+ .collect::<Result<_>>()?;++ *self.sort_order.lock() = vec![];++ let mut total_updated: u64 = 0;++ for partition_data in &self.batches {+ let mut partition = partition_data.write().await;+ let mut new_batches = Vec::with_capacity(partition.len());++ for batch in partition.iter() {+ if batch.num_rows() == 0 {+ continue;+ }++ // Evaluate filters - None means "match all rows"+ let filter_mask = evaluate_filters_to_mask(+ &filters,+ batch,+ &df_schema,+ state.execution_props(),+ )?;++ let (update_count, update_mask) = match filter_mask {+ Some(mask) => {+ // Count rows where mask is true (will be updated)+ let count = mask.iter().filter(|v| v == &Some(true)).count();+ // Normalize mask: only true (not NULL) triggers update+ let normalized: BooleanArray =+ mask.iter().map(|v| Some(v == Some(true))).collect();+ (count, normalized)+ }+ None => {+ // No filters = update all rows+ (+ batch.num_rows(),+ BooleanArray::from(vec![true; batch.num_rows()]),+ )+ }+ };++ total_updated += update_count as u64;++ if update_count == 0 {+ new_batches.push(batch.clone());+ continue;+ }++ let mut new_columns: Vec<ArrayRef> =+ Vec::with_capacity(batch.num_columns());++ for field in self.schema.fields() {+ let column_name = field.name();+ let original_column =+ batch.column_by_name(column_name).ok_or_else(|| {+ datafusion_common::DataFusionError::Internal(format!(+ "Column '{column_name}' not found in batch"+ ))+ })?;++ let new_column = if let Some(physical_expr) =+ physical_assignments.get(column_name.as_str())+ {+ // Use evaluate_selection to only evaluate on matching rows.+ // This avoids errors (e.g., divide-by-zero) on rows that won't+ // be updated. The result is scattered back with nulls for+ // non-matching rows, which zip() will replace with originals.+ let new_values =+ physical_expr.evaluate_selection(batch, &update_mask)?;+ let new_array = new_values.into_array(batch.num_rows())?;++ // Convert to &dyn Array which implements Datum+ let new_arr: &dyn Array = new_array.as_ref();+ let orig_arr: &dyn Array = original_column.as_ref();+ zip(&update_mask, &new_arr, &orig_arr)?+ } else {+ Arc::clone(original_column)+ };++ new_columns.push(new_column);+ }++ let updated_batch =+ ArrowRecordBatch::try_new(Arc::clone(&self.schema), new_columns)?;+ new_batches.push(updated_batch);+ }++ *partition = new_batches;+ }++ Ok(Arc::new(DmlResultExec::new(total_updated)))+ }+}++/// Evaluate filter expressions against a batch and return a combined boolean mask.+/// Returns None if filters is empty (meaning "match all rows").+/// The returned mask has true for rows that match the filter predicates.+fn evaluate_filters_to_mask(+ filters: &[Expr],+ batch: &RecordBatch,+ df_schema: &DFSchema,+ execution_props: &datafusion_expr::execution_props::ExecutionProps,+) -> Result<Option<BooleanArray>> {+ if filters.is_empty() {+ return Ok(None);+ }++ let mut combined_mask: Option<BooleanArray> = None;++ for filter_expr in filters {+ let physical_expr =+ create_physical_expr(filter_expr, df_schema, execution_props)?;++ let result = physical_expr.evaluate(batch)?;+ let array = result.into_array(batch.num_rows())?;+ let bool_array = array+ .as_any()+ .downcast_ref::<BooleanArray>()+ .ok_or_else(|| {+ datafusion_common::DataFusionError::Internal(+ "Filter did not evaluate to boolean".to_string(),+ )+ })?+ .clone();++ combined_mask = Some(match combined_mask {+ Some(existing) => and(&existing, &bool_array)?,+ None => bool_array,+ });+ }++ Ok(combined_mask)+}++/// Returns a single row with the count of affected rows.+#[derive(Debug)]+struct DmlResultExec {+ rows_affected: u64,+ schema: SchemaRef,+ properties: PlanProperties,+}++impl DmlResultExec {+ fn new(rows_affected: u64) -> Self {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "count",+ DataType::UInt64,+ false,+ )]));++ let properties = PlanProperties::new(+ datafusion_physical_expr::EquivalenceProperties::new(Arc::clone(&schema)),+ Partitioning::UnknownPartitioning(1),+ datafusion_physical_plan::execution_plan::EmissionType::Final,+ datafusion_physical_plan::execution_plan::Boundedness::Bounded,+ );++ Self {+ rows_affected,+ schema,+ properties,+ }+ }+}++impl DisplayAs for DmlResultExec {+ fn fmt_as(+ &self,+ t: DisplayFormatType,+ f: &mut std::fmt::Formatter,+ ) -> std::fmt::Result {+ match t {+ DisplayFormatType::Default+ | DisplayFormatType::Verbose+ | DisplayFormatType::TreeRender => {+ write!(f, "DmlResultExec: rows_affected={}", self.rows_affected)+ }+ }+ }+}++impl ExecutionPlan for DmlResultExec {+ fn name(&self) -> &str {+ "DmlResultExec"+ }++ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }++ fn properties(&self) -> &PlanProperties {+ &self.properties+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ _children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(self)+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<datafusion_execution::TaskContext>,+ ) -> Result<datafusion_execution::SendableRecordBatchStream> {+ // Create a single batch with the count+ let count_array = UInt64Array::from(vec![self.rows_affected]);+ let batch = ArrowRecordBatch::try_new(+ Arc::clone(&self.schema),+ vec![Arc::new(count_array) as ArrayRef],+ )?;++ // Create a stream that yields just this one batch+ let stream = futures::stream::iter(vec![Ok(batch)]);+ Ok(Box::pin(RecordBatchStreamAdapter::new(+ Arc::clone(&self.schema),+ stream,+ )))+ }}diff --git a/datafusion/catalog/src/table.rs b/datafusion/catalog/src/table.rsindex cabdb22c62..1f223852c2 100644--- a/datafusion/catalog/src/table.rs+++ b/datafusion/catalog/src/table.rs@@ -328,6 +328,31 @@ pub trait TableProvider: Debug + Sync + Send {) -> Result<Arc<dyn ExecutionPlan>> {not_impl_err!("Insert into not implemented for this table")}++ /// Delete rows matching the filter predicates.+ ///+ /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).+ /// Empty `filters` deletes all rows.+ async fn delete_from(+ &self,+ _state: &dyn Session,+ _filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ not_impl_err!("DELETE not supported for {} table", self.table_type())+ }++ /// Update rows matching the filter predicates.+ ///+ /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64).+ /// Empty `filters` updates all rows.+ async fn update(+ &self,+ _state: &dyn Session,+ _assignments: Vec<(String, Expr)>,+ _filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ not_impl_err!("UPDATE not supported for {} table", self.table_type())+ }}/// Arguments for scanning a table with [`TableProvider::scan_with_args`].diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 9eaf1403e5..cc7d534776 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -62,9 +62,12 @@ use arrow::compute::SortOptions;use arrow::datatypes::Schema;use arrow_schema::Field;use datafusion_catalog::ScanArgs;+use datafusion_common::Column;use datafusion_common::display::ToStringifiedPlan;use datafusion_common::format::ExplainAnalyzeLevel;-use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};+use datafusion_common::tree_node::{+ Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor,+};use datafusion_common::{DFSchema, ScalarValue, exec_err, internal_datafusion_err, internal_err, not_impl_err,plan_err,@@ -600,6 +603,58 @@ impl DefaultPhysicalPlanner {);}}+ LogicalPlan::Dml(DmlStatement {+ table_name,+ target,+ op: WriteOp::Delete,+ input,+ ..+ }) => {+ if let Some(provider) =+ target.as_any().downcast_ref::<DefaultTableSource>()+ {+ let filters = extract_dml_filters(input)?;+ provider+ .table_provider+ .delete_from(session_state, filters)+ .await+ .map_err(|e| {+ e.context(format!("DELETE operation on table '{table_name}'"))+ })?+ } else {+ return exec_err!(+ "Table source can't be downcasted to DefaultTableSource"+ );+ }+ }+ LogicalPlan::Dml(DmlStatement {+ table_name,+ target,+ op: WriteOp::Update,+ input,+ ..+ }) => {+ if let Some(provider) =+ target.as_any().downcast_ref::<DefaultTableSource>()+ {+ // For UPDATE, the assignments are encoded in the projection of input+ // We pass the filters and let the provider handle the projection+ let filters = extract_dml_filters(input)?;+ // Extract assignments from the projection in input plan+ let assignments = extract_update_assignments(input)?;+ provider+ .table_provider+ .update(session_state, assignments, filters)+ .await+ .map_err(|e| {+ e.context(format!("UPDATE operation on table '{table_name}'"))+ })?+ } else {+ return exec_err!(+ "Table source can't be downcasted to DefaultTableSource"+ );+ }+ }LogicalPlan::Window(Window { window_expr, .. }) => {assert_or_internal_err!(!window_expr.is_empty(),@@ -1846,6 +1901,107 @@ fn get_physical_expr_pair(Ok((physical_expr, physical_name))}+/// Extract filter predicates from a DML input plan (DELETE/UPDATE).+/// Walks the logical plan tree and collects Filter predicates,+/// splitting AND conjunctions into individual expressions.+/// Column qualifiers are stripped so expressions can be evaluated against+/// the TableProvider's schema.+///+fn extract_dml_filters(input: &Arc<LogicalPlan>) -> Result<Vec<Expr>> {+ let mut filters = Vec::new();++ input.apply(|node| {+ if let LogicalPlan::Filter(filter) = node {+ // Split AND predicates into individual expressions+ filters.extend(split_conjunction(&filter.predicate).into_iter().cloned());+ }+ Ok(TreeNodeRecursion::Continue)+ })?;++ // Strip table qualifiers from column references+ filters.into_iter().map(strip_column_qualifiers).collect()+}++/// Strip table qualifiers from column references in an expression.+/// This is needed because DML filter expressions contain qualified column names+/// (e.g., "table.column") but the TableProvider's schema only has simple names.+fn strip_column_qualifiers(expr: Expr) -> Result<Expr> {+ expr.transform(|e| {+ if let Expr::Column(col) = &e+ && col.relation.is_some()+ {+ // Strip the qualifier+ return Ok(Transformed::yes(Expr::Column(Column::new_unqualified(+ col.name.clone(),+ ))));+ }+ Ok(Transformed::no(e))+ })+ .map(|t| t.data)+}++/// Extract column assignments from an UPDATE input plan.+/// For UPDATE statements, the SQL planner encodes assignments as a projection+/// over the source table. This function extracts column name and expression pairs+/// from the projection. Column qualifiers are stripped from the expressions.+///+fn extract_update_assignments(input: &Arc<LogicalPlan>) -> Result<Vec<(String, Expr)>> {+ // The UPDATE input plan structure is:+ // Projection(updated columns as expressions with aliases)+ // Filter(optional WHERE clause)+ // TableScan+ //+ // Each projected expression has an alias matching the column name+ let mut assignments = Vec::new();++ // Find the top-level projection+ if let LogicalPlan::Projection(projection) = input.as_ref() {+ for expr in &projection.expr {+ if let Expr::Alias(alias) = expr {+ // The alias name is the column name being updated+ // The inner expression is the new value+ let column_name = alias.name.clone();+ // Only include if it's not just a column reference to itself+ // (those are columns that aren't being updated)+ if !is_identity_assignment(&alias.expr, &column_name) {+ // Strip qualifiers from the assignment expression+ let stripped_expr = strip_column_qualifiers((*alias.expr).clone())?;+ assignments.push((column_name, stripped_expr));+ }+ }+ }+ } else {+ // Try to find projection deeper in the plan+ input.apply(|node| {+ if let LogicalPlan::Projection(projection) = node {+ for expr in &projection.expr {+ if let Expr::Alias(alias) = expr {+ let column_name = alias.name.clone();+ if !is_identity_assignment(&alias.expr, &column_name) {+ let stripped_expr =+ strip_column_qualifiers((*alias.expr).clone())?;+ assignments.push((column_name, stripped_expr));+ }+ }+ }+ return Ok(TreeNodeRecursion::Stop);+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ }++ Ok(assignments)+}++/// Check if an assignment is an identity assignment (column = column)+/// These are columns that are not being modified in the UPDATE+fn is_identity_assignment(expr: &Expr, column_name: &str) -> bool {+ match expr {+ Expr::Column(col) => col.name == column_name,+ _ => false,+ }+}+/// Check if window bounds are valid after schema information is available, and/// window_frame bounds are casted to the corresponding column type./// queries like:@@ -3953,8 +4109,8 @@ digraph {let right = LogicalPlanBuilder::scan("right", source, None)?.build()?;let join_keys = (- vec![datafusion_common::Column::new(Some("left"), "a")],- vec![datafusion_common::Column::new(Some("right"), "a")],+ vec![Column::new(Some("left"), "a")],+ vec![Column::new(Some("right"), "a")],);let join = left.join(right, JoinType::Full, join_keys, None)?.build()?;diff --git a/datafusion/core/tests/custom_sources_cases/dml_planning.rs b/datafusion/core/tests/custom_sources_cases/dml_planning.rsnew file mode 100644index 0000000000..84cf97710a--- /dev/null+++ b/datafusion/core/tests/custom_sources_cases/dml_planning.rs@@ -0,0 +1,297 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Tests for DELETE and UPDATE planning to verify filter and assignment extraction.++use std::any::Any;+use std::sync::{Arc, Mutex};++use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use async_trait::async_trait;+use datafusion::datasource::{TableProvider, TableType};+use datafusion::error::Result;+use datafusion::execution::context::SessionContext;+use datafusion::logical_expr::Expr;+use datafusion_catalog::Session;+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::empty::EmptyExec;++/// A TableProvider that captures the filters passed to delete_from().+struct CaptureDeleteProvider {+ schema: SchemaRef,+ received_filters: Arc<Mutex<Option<Vec<Expr>>>>,+}++impl CaptureDeleteProvider {+ fn new(schema: SchemaRef) -> Self {+ Self {+ schema,+ received_filters: Arc::new(Mutex::new(None)),+ }+ }++ fn captured_filters(&self) -> Option<Vec<Expr>> {+ self.received_filters.lock().unwrap().clone()+ }+}++impl std::fmt::Debug for CaptureDeleteProvider {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("CaptureDeleteProvider")+ .field("schema", &self.schema)+ .finish()+ }+}++#[async_trait]+impl TableProvider for CaptureDeleteProvider {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _state: &dyn Session,+ _projection: Option<&Vec<usize>>,+ _filters: &[Expr],+ _limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(EmptyExec::new(Arc::clone(&self.schema))))+ }++ async fn delete_from(+ &self,+ _state: &dyn Session,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ *self.received_filters.lock().unwrap() = Some(filters);+ Ok(Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![+ Field::new("count", DataType::UInt64, false),+ ])))))+ }+}++/// A TableProvider that captures filters and assignments passed to update().+#[allow(clippy::type_complexity)]+struct CaptureUpdateProvider {+ schema: SchemaRef,+ received_filters: Arc<Mutex<Option<Vec<Expr>>>>,+ received_assignments: Arc<Mutex<Option<Vec<(String, Expr)>>>>,+}++impl CaptureUpdateProvider {+ fn new(schema: SchemaRef) -> Self {+ Self {+ schema,+ received_filters: Arc::new(Mutex::new(None)),+ received_assignments: Arc::new(Mutex::new(None)),+ }+ }++ fn captured_filters(&self) -> Option<Vec<Expr>> {+ self.received_filters.lock().unwrap().clone()+ }++ fn captured_assignments(&self) -> Option<Vec<(String, Expr)>> {+ self.received_assignments.lock().unwrap().clone()+ }+}++impl std::fmt::Debug for CaptureUpdateProvider {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("CaptureUpdateProvider")+ .field("schema", &self.schema)+ .finish()+ }+}++#[async_trait]+impl TableProvider for CaptureUpdateProvider {+ fn as_any(&self) -> &dyn Any {+ self+ }++ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _state: &dyn Session,+ _projection: Option<&Vec<usize>>,+ _filters: &[Expr],+ _limit: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(Arc::new(EmptyExec::new(Arc::clone(&self.schema))))+ }++ async fn update(+ &self,+ _state: &dyn Session,+ assignments: Vec<(String, Expr)>,+ filters: Vec<Expr>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ *self.received_filters.lock().unwrap() = Some(filters);+ *self.received_assignments.lock().unwrap() = Some(assignments);+ Ok(Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![+ Field::new("count", DataType::UInt64, false),+ ])))))+ }+}++fn test_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("status", DataType::Utf8, true),+ Field::new("value", DataType::Int32, true),+ ]))+}++#[tokio::test]+async fn test_delete_single_filter() -> Result<()> {+ let provider = Arc::new(CaptureDeleteProvider::new(test_schema()));+ let ctx = SessionContext::new();+ ctx.register_table("t", Arc::clone(&provider) as Arc<dyn TableProvider>)?;++ ctx.sql("DELETE FROM t WHERE id = 1")+ .await?+ .collect()+ .await?;++ let filters = provider+ .captured_filters()+ .expect("filters should be captured");+ assert_eq!(filters.len(), 1);+ assert!(filters[0].to_string().contains("id"));+ Ok(())+}++#[tokio::test]+async fn test_delete_multiple_filters() -> Result<()> {+ let provider = Arc::new(CaptureDeleteProvider::new(test_schema()));+ let ctx = SessionContext::new();+ ctx.register_table("t", Arc::clone(&provider) as Arc<dyn TableProvider>)?;++ ctx.sql("DELETE FROM t WHERE id = 1 AND status = 'x'")+ .await?+ .collect()+ .await?;++ let filters = provider+ .captured_filters()+ .expect("filters should be captured");+ assert!(!filters.is_empty());+ Ok(())+}++#[tokio::test]+async fn test_delete_no_filters() -> Result<()> {+ let provider = Arc::new(CaptureDeleteProvider::new(test_schema()));+ let ctx = SessionContext::new();+ ctx.register_table("t", Arc::clone(&provider) as Arc<dyn TableProvider>)?;++ ctx.sql("DELETE FROM t").await?.collect().await?;++ let filters = provider+ .captured_filters()+ .expect("filters should be captured");+ assert!(+ filters.is_empty(),+ "DELETE without WHERE should have empty filters"+ );+ Ok(())+}++#[tokio::test]+async fn test_delete_complex_expr() -> Result<()> {+ let provider = Arc::new(CaptureDeleteProvider::new(test_schema()));+ let ctx = SessionContext::new();+ ctx.register_table("t", Arc::clone(&provider) as Arc<dyn TableProvider>)?;++ ctx.sql("DELETE FROM t WHERE id > 5 AND (status = 'a' OR status = 'b')")+ .await?+ .collect()+ .await?;++ let filters = provider+ .captured_filters()+ .expect("filters should be captured");+ assert!(!filters.is_empty());+ Ok(())+}++#[tokio::test]+async fn test_update_assignments() -> Result<()> {+ let provider = Arc::new(CaptureUpdateProvider::new(test_schema()));+ let ctx = SessionContext::new();+ ctx.register_table("t", Arc::clone(&provider) as Arc<dyn TableProvider>)?;++ ctx.sql("UPDATE t SET value = 100, status = 'updated' WHERE id = 5")+ .await?+ .collect()+ .await?;++ let assignments = provider+ .captured_assignments()+ .expect("assignments should be captured");+ assert_eq!(assignments.len(), 2, "should have 2 assignments");++ let filters = provider+ .captured_filters()+ .expect("filters should be captured");+ assert!(!filters.is_empty(), "should have filter for WHERE clause");+ Ok(())+}++#[tokio::test]+async fn test_unsupported_table_delete() -> Result<()> {+ let schema = test_schema();+ let ctx = SessionContext::new();++ let empty_table = datafusion::datasource::empty::EmptyTable::new(schema);+ ctx.register_table("empty_t", Arc::new(empty_table))?;++ let result = ctx.sql("DELETE FROM empty_t WHERE id = 1").await;+ assert!(result.is_err() || result.unwrap().collect().await.is_err());+ Ok(())+}++#[tokio::test]+async fn test_unsupported_table_update() -> Result<()> {+ let schema = test_schema();+ let ctx = SessionContext::new();++ let empty_table = datafusion::datasource::empty::EmptyTable::new(schema);+ ctx.register_table("empty_t", Arc::new(empty_table))?;++ let result = ctx.sql("UPDATE empty_t SET value = 1 WHERE id = 1").await;++ assert!(result.is_err() || result.unwrap().collect().await.is_err());+ Ok(())+}diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex 7b6a3c5fbe..8453615c28 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -47,6 +47,7 @@ use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use async_trait::async_trait;use futures::stream::Stream;+mod dml_planning;mod provider_filter_pushdown;mod statistics;diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rsindex b02254bb7d..4fb78933d7 100644--- a/datafusion/expr/src/lib.rs+++ b/datafusion/expr/src/lib.rs@@ -62,6 +62,10 @@ pub mod interval_arithmetic {pub use datafusion_expr_common::interval_arithmetic::*;}pub mod logical_plan;+pub mod dml {+ //! DML (Data Manipulation Language) types for DELETE, UPDATE operations.+ pub use crate::logical_plan::dml::*;+}pub mod planner;pub mod registry;pub mod simplify;diff --git a/datafusion/expr/src/logical_plan/dml.rs b/datafusion/expr/src/logical_plan/dml.rsindex b8448a5da6..6ac3b309aa 100644--- a/datafusion/expr/src/logical_plan/dml.rs+++ b/datafusion/expr/src/logical_plan/dml.rs@@ -122,11 +122,9 @@ impl CopyTo {/// * `INSERT` - Appends new rows to the existing table. Calls/// [`TableProvider::insert_into`]///-/// * `DELETE` - Removes rows from the table. Currently NOT supported by the-/// [`TableProvider`] trait or builtin sources.+/// * `DELETE` - Removes rows from the table. Calls [`TableProvider::delete_from`]///-/// * `UPDATE` - Modifies existing rows in the table. Currently NOT supported by-/// the [`TableProvider`] trait or builtin sources.+/// * `UPDATE` - Modifies existing rows in the table. Calls [`TableProvider::update`]////// * `CREATE TABLE AS SELECT` - Creates a new table and populates it with data/// from a query. This is similar to the `INSERT` operation, but it creates a new@@ -136,6 +134,8 @@ impl CopyTo {////// [`TableProvider`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html/// [`TableProvider::insert_into`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.insert_into+/// [`TableProvider::delete_from`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.delete_from+/// [`TableProvider::update`]: https://docs.rs/datafusion/latest/datafusion/datasource/trait.TableProvider.html#method.update#[derive(Clone)]pub struct DmlStatement {/// The table namediff --git a/datafusion/sqllogictest/test_files/delete.slt b/datafusion/sqllogictest/test_files/delete.sltindex 258318f094..e86343b6bf 100644--- a/datafusion/sqllogictest/test_files/delete.slt+++ b/datafusion/sqllogictest/test_files/delete.slt@@ -34,7 +34,9 @@ explain delete from t1;logical_plan01)Dml: op=[Delete] table=[t1]02)--TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Delete)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0# Filtered by existing columns@@ -45,7 +47,9 @@ logical_plan01)Dml: op=[Delete] table=[t1]02)--Filter: CAST(t1.a AS Int64) = Int64(1) AND t1.b = CAST(Int64(2) AS Utf8View) AND t1.c > CAST(Int64(3) AS Float64) AND CAST(t1.d AS Int64) != Int64(4)03)----TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Delete)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0# Filtered by existing columns, using qualified and unqualified names@@ -56,7 +60,9 @@ logical_plan01)Dml: op=[Delete] table=[t1]02)--Filter: CAST(t1.a AS Int64) = Int64(1) AND t1.b = CAST(Int64(2) AS Utf8View) AND t1.c > CAST(Int64(3) AS Float64) AND CAST(t1.d AS Int64) != Int64(4)03)----TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Delete)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0# Filtered by a mix of columns and literal predicates@@ -67,7 +73,9 @@ logical_plan01)Dml: op=[Delete] table=[t1]02)--Filter: CAST(t1.a AS Int64) = Int64(1) AND Int64(1) = Int64(1) AND Boolean(true)03)----TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Delete)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0# Deleting by columns that do not exist returns an errordiff --git a/datafusion/sqllogictest/test_files/dml_delete.slt b/datafusion/sqllogictest/test_files/dml_delete.sltnew file mode 100644index 0000000000..3dae431ada--- /dev/null+++ b/datafusion/sqllogictest/test_files/dml_delete.slt@@ -0,0 +1,202 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+## DELETE tests for MemTable+##########++# Test basic DELETE with WHERE clause+statement ok+CREATE TABLE test_delete AS VALUES (1, 'a'), (2, 'b'), (3, 'c');++query I+DELETE FROM test_delete WHERE column1 > 1;+----+2++query IT rowsort+SELECT * FROM test_delete;+----+1 a++statement ok+DROP TABLE test_delete;++# Test DELETE all rows (no WHERE clause)+statement ok+CREATE TABLE test_delete_all AS VALUES (1, 'x'), (2, 'y'), (3, 'z');++query I+DELETE FROM test_delete_all;+----+3++query I+SELECT COUNT(*) FROM test_delete_all;+----+0++statement ok+DROP TABLE test_delete_all;++# Test DELETE with compound predicate (AND)+statement ok+CREATE TABLE test_delete_compound AS VALUES (1, 10), (2, 20), (3, 30), (4, 40);++query I+DELETE FROM test_delete_compound WHERE column1 > 1 AND column2 < 40;+----+2++query II rowsort+SELECT * FROM test_delete_compound;+----+1 10+4 40++statement ok+DROP TABLE test_delete_compound;++# Test DELETE with OR predicate+statement ok+CREATE TABLE test_delete_or AS VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd');++query I+DELETE FROM test_delete_or WHERE column1 = 1 OR column1 = 4;+----+2++query IT rowsort+SELECT * FROM test_delete_or;+----+2 b+3 c++statement ok+DROP TABLE test_delete_or;++# Test DELETE with no matching rows+statement ok+CREATE TABLE test_delete_nomatch AS VALUES (1, 'a'), (2, 'b');++query I+DELETE FROM test_delete_nomatch WHERE column1 > 100;+----+0++query IT rowsort+SELECT * FROM test_delete_nomatch;+----+1 a+2 b++statement ok+DROP TABLE test_delete_nomatch;++# Test DELETE with IS NULL predicate+statement ok+CREATE TABLE test_delete_null(id INT, name VARCHAR);++statement ok+INSERT INTO test_delete_null VALUES (1, 'one'), (2, NULL), (3, 'three');++query I+DELETE FROM test_delete_null WHERE name IS NULL;+----+1++query IT rowsort+SELECT * FROM test_delete_null;+----+1 one+3 three++statement ok+DROP TABLE test_delete_null;++# Test DELETE with NULL predicate (SQL three-valued logic)+# When comparing with NULL, the predicate evaluates to NULL, not true/false+# Rows where predicate is NULL should NOT be deleted+statement ok+CREATE TABLE test_delete_null_pred(id INT, value INT);++statement ok+INSERT INTO test_delete_null_pred VALUES (1, 10), (2, NULL), (3, 30);++# This predicate evaluates to NULL for row with id=2 (because NULL > 15 is NULL)+# Only row with id=3 should be deleted (30 > 15 is true)+# Row with id=2 should be kept (NULL > 15 is NULL, not true)+query I+DELETE FROM test_delete_null_pred WHERE value > 15;+----+1++query II rowsort+SELECT * FROM test_delete_null_pred;+----+1 10+2 NULL++statement ok+DROP TABLE test_delete_null_pred;++# Test multiple DELETEs on same table (state persistence)+statement ok+CREATE TABLE test_multi_delete AS VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e');++query I+DELETE FROM test_multi_delete WHERE column1 = 1;+----+1++query I+DELETE FROM test_multi_delete WHERE column1 = 3;+----+1++query I+DELETE FROM test_multi_delete WHERE column1 = 5;+----+1++query IT rowsort+SELECT * FROM test_multi_delete;+----+2 b+4 d++statement ok+DROP TABLE test_multi_delete;++# Test DELETE with IN predicate+statement ok+CREATE TABLE test_delete_in AS VALUES (1), (2), (3), (4), (5);++query I+DELETE FROM test_delete_in WHERE column1 IN (2, 4);+----+2++query I rowsort+SELECT * FROM test_delete_in;+----+1+3+5++statement ok+DROP TABLE test_delete_in;diff --git a/datafusion/sqllogictest/test_files/dml_update.slt b/datafusion/sqllogictest/test_files/dml_update.sltnew file mode 100644index 0000000000..10f74ae397--- /dev/null+++ b/datafusion/sqllogictest/test_files/dml_update.slt@@ -0,0 +1,286 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at++# http://www.apache.org/licenses/LICENSE-2.0++# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++##########+## UPDATE tests for MemTable+##########++# Test basic UPDATE with WHERE clause+statement ok+CREATE TABLE test_update AS VALUES (1, 'a'), (2, 'b'), (3, 'c');++query I+UPDATE test_update SET column2 = 'updated' WHERE column1 = 2;+----+1++query IT rowsort+SELECT * FROM test_update;+----+1 a+2 updated+3 c++statement ok+DROP TABLE test_update;++# Test UPDATE all rows (no WHERE clause)+statement ok+CREATE TABLE test_update_all AS VALUES (1, 'x'), (2, 'y'), (3, 'z');++query I+UPDATE test_update_all SET column2 = 'changed';+----+3++query IT rowsort+SELECT * FROM test_update_all;+----+1 changed+2 changed+3 changed++statement ok+DROP TABLE test_update_all;++# Test UPDATE multiple columns+statement ok+CREATE TABLE test_update_multi(id INT, name VARCHAR, value INT);++statement ok+INSERT INTO test_update_multi VALUES (1, 'one', 10), (2, 'two', 20), (3, 'three', 30);++query I+UPDATE test_update_multi SET name = 'updated', value = 99 WHERE id = 2;+----+1++query ITI rowsort+SELECT * FROM test_update_multi;+----+1 one 10+2 updated 99+3 three 30++statement ok+DROP TABLE test_update_multi;++# Test UPDATE with compound predicate (AND)+statement ok+CREATE TABLE test_update_compound AS VALUES (1, 10), (2, 20), (3, 30), (4, 40);++query I+UPDATE test_update_compound SET column2 = 0 WHERE column1 > 1 AND column2 < 40;+----+2++query II rowsort+SELECT * FROM test_update_compound;+----+1 10+2 0+3 0+4 40++statement ok+DROP TABLE test_update_compound;++# Test UPDATE with OR predicate+statement ok+CREATE TABLE test_update_or AS VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd');++query I+UPDATE test_update_or SET column2 = 'modified' WHERE column1 = 1 OR column1 = 4;+----+2++query IT rowsort+SELECT * FROM test_update_or;+----+1 modified+2 b+3 c+4 modified++statement ok+DROP TABLE test_update_or;++# Test UPDATE with no matching rows+statement ok+CREATE TABLE test_update_nomatch AS VALUES (1, 'a'), (2, 'b');++query I+UPDATE test_update_nomatch SET column2 = 'new' WHERE column1 > 100;+----+0++query IT rowsort+SELECT * FROM test_update_nomatch;+----+1 a+2 b++statement ok+DROP TABLE test_update_nomatch;++# Test UPDATE with NULL predicate (SQL three-valued logic)+# Rows where predicate is NULL should NOT be updated+statement ok+CREATE TABLE test_update_null_pred(id INT, value INT, name VARCHAR);++statement ok+INSERT INTO test_update_null_pred VALUES (1, 10, 'a'), (2, NULL, 'b'), (3, 30, 'c');++# This predicate evaluates to NULL for row with id=2 (because NULL > 15 is NULL)+# Only row with id=3 should be updated (30 > 15 is true)+# Row with id=2 should keep its original name 'b' (NULL > 15 is NULL, not true)+query I+UPDATE test_update_null_pred SET name = 'updated' WHERE value > 15;+----+1++query IIT rowsort+SELECT * FROM test_update_null_pred;+----+1 10 a+2 NULL b+3 30 updated++statement ok+DROP TABLE test_update_null_pred;++# Test UPDATE with arithmetic expression (SET column = column * 2)+statement ok+CREATE TABLE test_update_expr AS VALUES (1, 10), (2, 20), (3, 30);++query I+UPDATE test_update_expr SET column2 = column2 * 2 WHERE column1 = 2;+----+1++query II rowsort+SELECT * FROM test_update_expr;+----+1 10+2 40+3 30++statement ok+DROP TABLE test_update_expr;++# Test UPDATE setting column to NULL+statement ok+CREATE TABLE test_update_null(id INT, name VARCHAR);++statement ok+INSERT INTO test_update_null VALUES (1, 'one'), (2, 'two'), (3, 'three');++query I+UPDATE test_update_null SET name = NULL WHERE id = 2;+----+1++query IT rowsort+SELECT * FROM test_update_null;+----+1 one+2 NULL+3 three++statement ok+DROP TABLE test_update_null;++# Test UPDATE with CASE expression in SET+statement ok+CREATE TABLE test_update_case AS VALUES (1, 'low'), (50, 'medium'), (100, 'high');++query I+UPDATE test_update_case SET column2 = CASE+ WHEN column1 < 25 THEN 'small'+ WHEN column1 < 75 THEN 'medium'+ ELSE 'large'+END;+----+3++query IT rowsort+SELECT * FROM test_update_case;+----+1 small+100 large+50 medium++statement ok+DROP TABLE test_update_case;++# Test UPDATE with column reference (SET a = b)+statement ok+CREATE TABLE test_update_col_ref AS VALUES (1, 10, 100), (2, 20, 200);++query I+UPDATE test_update_col_ref SET column1 = column2 WHERE column3 = 100;+----+1++query III rowsort+SELECT * FROM test_update_col_ref;+----+10 10 100+2 20 200++statement ok+DROP TABLE test_update_col_ref;++# Test UPDATE with invalid column name (error case)+statement ok+CREATE TABLE test_update_error(id INT, name VARCHAR);++statement ok+INSERT INTO test_update_error VALUES (1, 'test');++statement error No field named nonexistent+UPDATE test_update_error SET nonexistent = 'value';++statement ok+DROP TABLE test_update_error;++# Test UPDATE with expression that would error on non-matching rows+# Regression test: expressions should only be evaluated on rows that match+# the WHERE clause, not all rows. This prevents divide-by-zero errors+# on rows that won't be updated.+statement ok+CREATE TABLE test_update_div(id INT, divisor INT, result INT);++statement ok+INSERT INTO test_update_div VALUES (1, 0, 0), (2, 2, 0), (3, 5, 0);++# This should succeed: 1/divisor is only evaluated where divisor != 0+# Row 1 (divisor=0) is excluded by WHERE clause and expression is NOT evaluated+query I+UPDATE test_update_div SET result = 100 / divisor WHERE divisor != 0;+----+2++query III rowsort+SELECT * FROM test_update_div;+----+1 0 0+2 2 50+3 5 20++statement ok+DROP TABLE test_update_div;diff --git a/datafusion/sqllogictest/test_files/update.slt b/datafusion/sqllogictest/test_files/update.sltindex 9f2c16b211..a652ae7633 100644--- a/datafusion/sqllogictest/test_files/update.slt+++ b/datafusion/sqllogictest/test_files/update.slt@@ -33,7 +33,9 @@ logical_plan01)Dml: op=[Update] table=[t1]02)--Projection: CAST(Int64(1) AS Int32) AS a, CAST(Int64(2) AS Utf8View) AS b, Float64(3) AS c, CAST(NULL AS Int32) AS d03)----TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Update)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0query TTexplain update t1 set a=c+1, b=a, c=c+1.0, d=b;@@ -42,7 +44,9 @@ logical_plan01)Dml: op=[Update] table=[t1]02)--Projection: CAST(t1.c + CAST(Int64(1) AS Float64) AS Int32) AS a, CAST(t1.a AS Utf8View) AS b, t1.c + Float64(1) AS c, CAST(t1.b AS Int32) AS d03)----TableScan: t1-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Update)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0statement okcreate table t2(a int, b varchar, c double, d int);@@ -70,10 +74,12 @@ logical_plan01)Dml: op=[Update] table=[t1]02)--Projection: t1.a AS a, t2.b AS b, CAST(t2.a AS Float64) AS c, CAST(Int64(1) AS Int32) AS d03)----Filter: t1.a = t2.a AND t1.b > CAST(Utf8("foo") AS Utf8View) AND t2.c > Float64(1)-04)------Cross Join:+04)------Cross Join:05)--------TableScan: t106)--------TableScan: t2-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Update)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0statement okcreate table t3(a int, b varchar, c double, d int);@@ -90,8 +96,10 @@ logical_plan01)Dml: op=[Update] table=[t1]02)--Projection: t.a AS a, t2.b AS b, CAST(t.a AS Float64) AS c, CAST(Int64(1) AS Int32) AS d03)----Filter: t.a = t2.a AND t.b > CAST(Utf8("foo") AS Utf8View) AND t2.c > Float64(1)-04)------Cross Join:+04)------Cross Join:05)--------SubqueryAlias: t06)----------TableScan: t107)--------TableScan: t2-physical_plan_error This feature is not implemented: Unsupported logical plan: Dml(Update)+physical_plan+01)CooperativeExec+02)--DmlResultExec: rows_affected=0