Apache DataFusion SQL Query Engine
Provide session to the udtf call (#20222)
Albert Skalt5ff80e434087
diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rsindex 67f3dc2826..ae890f7c95 100644--- a/datafusion-cli/src/functions.rs+++ b/datafusion-cli/src/functions.rs@@ -31,7 +31,7 @@ use arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer};use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef, TimeUnit};use arrow::record_batch::RecordBatch;use arrow::util::pretty::pretty_format_batches;-use datafusion::catalog::{Session, TableFunctionImpl};+use datafusion::catalog::{Session, TableFunctionArgs, TableFunctionImpl};use datafusion::common::{Column, plan_err};use datafusion::datasource::TableProvider;use datafusion::datasource::memory::MemorySourceConfig;@@ -326,7 +326,8 @@ fn fixed_len_byte_array_to_string(val: Option<&FixedLenByteArray>) -> Option<Strpub struct ParquetMetadataFunc {}impl TableFunctionImpl for ParquetMetadataFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();let filename = match exprs.first() {Some(Expr::Literal(ScalarValue::Utf8(Some(s)), _)) => s, // single quote: parquet_metadata('x.parquet')Some(Expr::Column(Column { name, .. })) => name, // double quote: parquet_metadata("x.parquet")@@ -517,7 +518,8 @@ impl MetadataCacheFunc {}impl TableFunctionImpl for MetadataCacheFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();if !exprs.is_empty() {return plan_err!("metadata_cache should have no arguments");}@@ -635,7 +637,8 @@ impl StatisticsCacheFunc {}impl TableFunctionImpl for StatisticsCacheFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();if !exprs.is_empty() {return plan_err!("statistics_cache should have no arguments");}@@ -770,7 +773,8 @@ impl ListFilesCacheFunc {}impl TableFunctionImpl for ListFilesCacheFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();if !exprs.is_empty() {return plan_err!("list_files_cache should have no arguments");}diff --git a/datafusion-examples/README.md b/datafusion-examples/README.mdindex 496d16527c..073f269d4a 100644--- a/datafusion-examples/README.md+++ b/datafusion-examples/README.md@@ -218,13 +218,14 @@ cargo run --example dataframe -- dataframe#### Category: Single Process-| Subcommand | File Path | Description |-| ---------- | ------------------------------------------------------- | ----------------------------------------------- |-| adv_udaf | [`udf/advanced_udaf.rs`](examples/udf/advanced_udaf.rs) | Advanced User Defined Aggregate Function (UDAF) |-| adv_udf | [`udf/advanced_udf.rs`](examples/udf/advanced_udf.rs) | Advanced User Defined Scalar Function (UDF) |-| adv_udwf | [`udf/advanced_udwf.rs`](examples/udf/advanced_udwf.rs) | Advanced User Defined Window Function (UDWF) |-| async_udf | [`udf/async_udf.rs`](examples/udf/async_udf.rs) | Asynchronous User Defined Scalar Function |-| udaf | [`udf/simple_udaf.rs`](examples/udf/simple_udaf.rs) | Simple UDAF example |-| udf | [`udf/simple_udf.rs`](examples/udf/simple_udf.rs) | Simple UDF example |-| udtf | [`udf/simple_udtf.rs`](examples/udf/simple_udtf.rs) | Simple UDTF example |-| udwf | [`udf/simple_udwf.rs`](examples/udf/simple_udwf.rs) | Simple UDWF example |+| Subcommand | File Path | Description |+| --------------- | ----------------------------------------------------------- | ----------------------------------------------- |+| adv_udaf | [`udf/advanced_udaf.rs`](examples/udf/advanced_udaf.rs) | Advanced User Defined Aggregate Function (UDAF) |+| adv_udf | [`udf/advanced_udf.rs`](examples/udf/advanced_udf.rs) | Advanced User Defined Scalar Function (UDF) |+| adv_udwf | [`udf/advanced_udwf.rs`](examples/udf/advanced_udwf.rs) | Advanced User Defined Window Function (UDWF) |+| async_udf | [`udf/async_udf.rs`](examples/udf/async_udf.rs) | Asynchronous User Defined Scalar Function |+| udaf | [`udf/simple_udaf.rs`](examples/udf/simple_udaf.rs) | Simple UDAF example |+| udf | [`udf/simple_udf.rs`](examples/udf/simple_udf.rs) | Simple UDF example |+| udtf | [`udf/simple_udtf.rs`](examples/udf/simple_udtf.rs) | Simple UDTF example |+| udwf | [`udf/simple_udwf.rs`](examples/udf/simple_udwf.rs) | Simple UDWF example |+| table_list_udtf | [`udf/table_list_udtf.rs`](examples/udf/table_list_udtf.rs) | Session-aware UDTF table list example |diff --git a/datafusion-examples/examples/udf/main.rs b/datafusion-examples/examples/udf/main.rsindex e024e466ab..89f3fd801d 100644--- a/datafusion-examples/examples/udf/main.rs+++ b/datafusion-examples/examples/udf/main.rs@@ -21,7 +21,7 @@//!//! ## Usage//! ```bash-//! cargo run --example udf -- [all|adv_udaf|adv_udf|adv_udwf|async_udf|udaf|udf|udtf|udwf]+//! cargo run --example udf -- [all|adv_udaf|adv_udf|adv_udwf|async_udf|udaf|udf|udtf|udwf|table_list_udtf]//! ```//!//! Each subcommand runs a corresponding example:@@ -50,6 +50,9 @@//!//! - `udwf`//! (file: simple_udwf.rs, desc: Simple UDWF example)+//!+//! - `table_list_udtf`+//! (file: table_list_udtf.rs, desc: Session-aware UDTF table list example)mod advanced_udaf;mod advanced_udf;@@ -59,6 +62,7 @@ mod simple_udaf;mod simple_udf;mod simple_udtf;mod simple_udwf;+mod table_list_udtf;use datafusion::error::{DataFusionError, Result};use strum::{IntoEnumIterator, VariantNames};@@ -76,6 +80,7 @@ enum ExampleKind {Udaf,Udwf,Udtf,+ TableListUdtf,}impl ExampleKind {@@ -101,6 +106,7 @@ impl ExampleKind {ExampleKind::Udf => simple_udf::simple_udf().await?,ExampleKind::Udtf => simple_udtf::simple_udtf().await?,ExampleKind::Udwf => simple_udwf::simple_udwf().await?,+ ExampleKind::TableListUdtf => table_list_udtf::table_list_udtf().await?,}Ok(())diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rsindex ee2615c4a5..b00a06d301 100644--- a/datafusion-examples/examples/udf/simple_udtf.rs+++ b/datafusion-examples/examples/udf/simple_udtf.rs@@ -27,7 +27,7 @@ use arrow::csv::reader::Format;use async_trait::async_trait;use datafusion::arrow::datatypes::SchemaRef;use datafusion::arrow::record_batch::RecordBatch;-use datafusion::catalog::{Session, TableFunctionImpl};+use datafusion::catalog::{Session, TableFunctionArgs, TableFunctionImpl};use datafusion::common::{ScalarValue, plan_err};use datafusion::datasource::TableProvider;use datafusion::datasource::memory::MemorySourceConfig;@@ -135,7 +135,8 @@ impl TableProvider for LocalCsvTable {struct LocalCsvTableFunc {}impl TableFunctionImpl for LocalCsvTableFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();let Some(Expr::Literal(ScalarValue::Utf8(Some(path)), _)) = exprs.first() else {return plan_err!("read_csv requires at least one string argument");};diff --git a/datafusion-examples/examples/udf/table_list_udtf.rs b/datafusion-examples/examples/udf/table_list_udtf.rsnew file mode 100644index 0000000000..739f8e11f0--- /dev/null+++ b/datafusion-examples/examples/udf/table_list_udtf.rs@@ -0,0 +1,128 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! See `main.rs` for how to run it.++use std::sync::{Arc, LazyLock};++use arrow::array::{RecordBatch, StringBuilder};+use arrow_schema::{DataType, Field, Schema, SchemaRef};+use datafusion::{+ catalog::{MemTable, TableFunctionArgs, TableFunctionImpl, TableProvider},+ common::Result,+ execution::SessionState,+ prelude::SessionContext,+};+use datafusion_common::{DataFusionError, plan_err};+use tokio::{runtime::Handle, task::block_in_place};++const FUNCTION_NAME: &str = "table_list";++// The example shows, how to create UDTF that depends on the session state.+// Defines a `table_list` UDTF that returns a list of tables within the provided session.++pub async fn table_list_udtf() -> Result<()> {+ let ctx = SessionContext::new();+ ctx.register_udtf(FUNCTION_NAME, Arc::new(TableListUdtf));++ // Register different kinds of tables.+ ctx.sql("create view v as select 1")+ .await?+ .collect()+ .await?;+ ctx.sql("create table t(a int)").await?.collect().await?;++ // Print results.+ ctx.sql("select * from table_list()").await?.show().await?;++ Ok(())+}++#[derive(Debug, Default)]+struct TableListUdtf;++static SCHEMA: LazyLock<SchemaRef> = LazyLock::new(|| {+ SchemaRef::new(Schema::new(vec![+ Field::new("catalog", DataType::Utf8, false),+ Field::new("schema", DataType::Utf8, false),+ Field::new("table", DataType::Utf8, false),+ Field::new("type", DataType::Utf8, false),+ ]))+});++impl TableFunctionImpl for TableListUdtf {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ if !args.exprs().is_empty() {+ return plan_err!(+ "{}: unexpected number of arguments: {}, expected: 0",+ FUNCTION_NAME,+ args.exprs().len()+ );+ }+ let state = args+ .session()+ .as_any()+ .downcast_ref::<SessionState>()+ .ok_or_else(|| {+ DataFusionError::Internal("failed to downcast state".into())+ })?;++ let mut catalogs = StringBuilder::new();+ let mut schemas = StringBuilder::new();+ let mut tables = StringBuilder::new();+ let mut types = StringBuilder::new();++ let catalog_list = state.catalog_list();+ for catalog_name in catalog_list.catalog_names() {+ let Some(catalog) = catalog_list.catalog(&catalog_name) else {+ continue;+ };+ for schema_name in catalog.schema_names() {+ let Some(schema) = catalog.schema(&schema_name) else {+ continue;+ };+ for table_name in schema.table_names() {+ let Some(provider) = block_in_place(|| {+ Handle::current().block_on(schema.table(&table_name))+ })?+ else {+ continue;+ };+ catalogs.append_value(catalog_name.clone());+ schemas.append_value(schema_name.clone());+ tables.append_value(table_name.clone());+ types.append_value(provider.table_type().to_string())+ }+ }+ }++ let batch = RecordBatch::try_new(+ Arc::clone(&SCHEMA),+ vec![+ Arc::new(catalogs.finish()),+ Arc::new(schemas.finish()),+ Arc::new(tables.finish()),+ Arc::new(types.finish()),+ ],+ )?;++ Ok(Arc::new(MemTable::try_new(+ batch.schema(),+ vec![vec![batch]],+ )?))+ }+}diff --git a/datafusion/catalog/src/table.rs b/datafusion/catalog/src/table.rsindex 361589c5b6..4d939b883f 100644--- a/datafusion/catalog/src/table.rs+++ b/datafusion/catalog/src/table.rs@@ -23,8 +23,8 @@ use std::sync::Arc;use crate::session::Session;use arrow::datatypes::SchemaRef;use async_trait::async_trait;-use datafusion_common::Result;use datafusion_common::{Constraints, Statistics, not_impl_err};+use datafusion_common::{Result, internal_err};use datafusion_expr::Expr;use datafusion_expr::dml::InsertOp;@@ -507,10 +507,49 @@ pub trait TableProviderFactory: Debug + Sync + Send {) -> Result<Arc<dyn TableProvider>>;}+/// Describes arguments provided to the table function call.+pub struct TableFunctionArgs<'e, 's> {+ /// Call arguments.+ exprs: &'e [Expr],+ /// Session within which the function is called.+ session: &'s dyn Session,+}++impl<'e, 's> TableFunctionArgs<'e, 's> {+ /// Make a new [`TableFunctionArgs`].+ pub fn new(exprs: &'e [Expr], session: &'s dyn Session) -> Self {+ Self { exprs, session }+ }++ /// Get expressions passed as the called function arguments.+ pub fn exprs(&self) -> &'e [Expr] {+ self.exprs+ }++ /// Get a session where the table function is called.+ pub fn session(&self) -> &'s dyn Session {+ self.session+ }+}+/// A trait for table function implementationspub trait TableFunctionImpl: Debug + Sync + Send + Any {/// Create a table provider- fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>>;+ #[deprecated(+ since = "53.0.0",+ note = "Implement `TableFunctionImpl::call_with_args` instead"+ )]+ fn call(&self, _exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ internal_err!(+ "TableFunctionImpl::call is not implemented. Implement TableFunctionImpl::call_with_args instead."+ )+ }++ /// Create a table provider+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ #[expect(deprecated)]+ self.call(args.exprs)+ }}/// A table that uses a function to generate data@@ -539,7 +578,20 @@ impl TableFunction {}/// Get the function implementation and generate a table+ #[deprecated(+ since = "53.0.0",+ note = "Use `TableFunction::create_table_provider_with_args` instead"+ )]pub fn create_table_provider(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ #[expect(deprecated)]self.fun.call(args)}++ /// Get the function implementation and generate a table+ pub fn create_table_provider_with_args(+ &self,+ args: TableFunctionArgs,+ ) -> Result<Arc<dyn TableProvider>> {+ self.fun.call_with_args(args)+ }}diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 6ff5d14762..f0888e0104 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -1862,6 +1862,8 @@ impl ContextProvider for SessionContextProvider<'_> {name: &str,args: Vec<Expr>,) -> datafusion_common::Result<Arc<dyn TableSource>> {+ use datafusion_catalog::TableFunctionArgs;+let tbl_func = self.state.table_functions@@ -1884,7 +1886,8 @@ impl ContextProvider for SessionContextProvider<'_> {.and_then(|e| simplifier.simplify(e))}).collect::<datafusion_common::Result<Vec<_>>>()?;- let provider = tbl_func.create_table_provider(&args)?;+ let provider = tbl_func+ .create_table_provider_with_args(TableFunctionArgs::new(&args, self.state))?;Ok(provider_as_source(provider))}diff --git a/datafusion/core/tests/user_defined/user_defined_table_functions.rs b/datafusion/core/tests/user_defined/user_defined_table_functions.rsindex 95694d00a6..2a0e85068e 100644--- a/datafusion/core/tests/user_defined/user_defined_table_functions.rs+++ b/datafusion/core/tests/user_defined/user_defined_table_functions.rs@@ -33,8 +33,8 @@ use datafusion::error::Result;use datafusion::execution::TaskContext;use datafusion::physical_plan::{ExecutionPlan, collect};use datafusion::prelude::SessionContext;-use datafusion_catalog::Session;use datafusion_catalog::TableFunctionImpl;+use datafusion_catalog::{Session, TableFunctionArgs};use datafusion_common::{DFSchema, ScalarValue};use datafusion_expr::{EmptyRelation, Expr, LogicalPlan, Projection, TableType};@@ -200,7 +200,8 @@ impl SimpleCsvTable {struct SimpleCsvTableFunc {}impl TableFunctionImpl for SimpleCsvTableFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();let mut new_exprs = vec![];let mut filepath = String::new();for expr in exprs {@@ -231,7 +232,7 @@ async fn test_udtf_type_coercion() -> Result<()> {struct NoOpTableFunc;impl TableFunctionImpl for NoOpTableFunc {- fn call(&self, _: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, _: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {let schema = Arc::new(arrow::datatypes::Schema::empty());Ok(Arc::new(MemTable::try_new(schema, vec![vec![]])?))}diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rsindex 35c13c1169..ce5422f0c9 100644--- a/datafusion/ffi/src/udtf.rs+++ b/datafusion/ffi/src/udtf.rs@@ -21,21 +21,23 @@ use std::sync::Arc;use abi_stable::StableAbi;use abi_stable::std_types::{RResult, RVec};-use datafusion_catalog::{TableFunctionImpl, TableProvider};+use datafusion_catalog::{TableFunctionArgs, TableFunctionImpl, TableProvider};+use datafusion_common::DataFusionError;use datafusion_common::error::Result;use datafusion_execution::TaskContext;-use datafusion_expr::Expr;use datafusion_proto::logical_plan::from_proto::parse_exprs;use datafusion_proto::logical_plan::to_proto::serialize_exprs;use datafusion_proto::logical_plan::{DefaultLogicalExtensionCodec, LogicalExtensionCodec,};use datafusion_proto::protobuf::LogicalExprList;+use datafusion_session::Session;use prost::Message;use tokio::runtime::Handle;use crate::execution::FFI_TaskContextProvider;use crate::proto::logical_extension_codec::FFI_LogicalExtensionCodec;+use crate::session::{FFI_SessionRef, ForeignSession};use crate::table_provider::FFI_TableProvider;use crate::util::FFIResult;use crate::{df_result, rresult_return};@@ -44,11 +46,22 @@ use crate::{df_result, rresult_return};#[repr(C)]#[derive(Debug, StableAbi)]pub struct FFI_TableFunction {- /// Equivalent to the `call` function of the TableFunctionImpl.+ /// Equivalent to the [`TableFunctionImpl::call`]./// The arguments are Expr passed as protobuf encoded bytes.+ #[deprecated(+ since = "53.0.0",+ note = "See TableFunctionImpl::call deprecation note"+ )]pub call:unsafe extern "C" fn(udtf: &Self, args: RVec<u8>) -> FFIResult<FFI_TableProvider>,+ /// Equivalent to the [`TableFunctionImpl::call_with_args`].+ call_with_args: unsafe extern "C" fn(+ udtf: &Self,+ args: RVec<u8>,+ session: FFI_SessionRef,+ ) -> FFIResult<FFI_TableProvider>,+pub logical_codec: FFI_LogicalExtensionCodec,/// Used to create a clone on the provider of the udtf. This should@@ -107,6 +120,7 @@ unsafe extern "C" fn call_fn_wrapper(codec.as_ref()));+ #[expect(deprecated)]let table_provider = rresult_return!(udtf_inner.call(&args));RResult::ROk(FFI_TableProvider::new_with_ffi_codec(table_provider,@@ -116,6 +130,47 @@ unsafe extern "C" fn call_fn_wrapper())}+unsafe extern "C" fn call_with_args_wrapper(+ udtf: &FFI_TableFunction,+ args: RVec<u8>,+ session: FFI_SessionRef,+) -> FFIResult<FFI_TableProvider> {+ let runtime = udtf.runtime();+ let udtf_inner = udtf.inner();++ let ctx: Arc<TaskContext> =+ rresult_return!((&udtf.logical_codec.task_ctx_provider).try_into());+ let codec: Arc<dyn LogicalExtensionCodec> = (&udtf.logical_codec).into();++ let proto_filters = rresult_return!(LogicalExprList::decode(args.as_ref()));++ let args = rresult_return!(parse_exprs(+ proto_filters.expr.iter(),+ ctx.as_ref(),+ codec.as_ref()+ ));++ let mut foreign_session = None;+ let session = rresult_return!(+ session+ .as_local()+ .map(Ok::<&(dyn Session + Send + Sync), DataFusionError>)+ .unwrap_or_else(|| {+ foreign_session = Some(ForeignSession::try_from(&session)?);+ Ok(foreign_session.as_ref().unwrap())+ })+ );+ let table_provider = rresult_return!(+ udtf_inner.call_with_args(TableFunctionArgs::new(&args, session))+ );+ RResult::ROk(FFI_TableProvider::new_with_ffi_codec(+ table_provider,+ false,+ runtime,+ udtf.logical_codec.clone(),+ ))+}+unsafe extern "C" fn release_fn_wrapper(udtf: &mut FFI_TableFunction) {unsafe {debug_assert!(!udtf.private_data.is_null());@@ -176,7 +231,9 @@ impl FFI_TableFunction {let private_data = Box::new(TableFunctionPrivateData { udtf, runtime });Self {+ #[expect(deprecated)]call: call_fn_wrapper,+ call_with_args: call_with_args_wrapper,logical_codec,clone: clone_fn_wrapper,release: release_fn_wrapper,@@ -215,13 +272,35 @@ impl From<FFI_TableFunction> for Arc<dyn TableFunctionImpl> {}impl TableFunctionImpl for ForeignTableFunction {- fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let session = FFI_SessionRef::new(+ args.session(),+ self.0.runtime(),+ self.0.logical_codec.clone(),+ );+ let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();+ let expr_list = LogicalExprList {+ expr: serialize_exprs(args.exprs(), codec.as_ref())?,+ };+ let filters_serialized = expr_list.encode_to_vec().into();++ let table_provider =+ unsafe { (self.0.call_with_args)(&self.0, filters_serialized, session) };++ let table_provider = df_result!(table_provider)?;+ let table_provider: Arc<dyn TableProvider> = (&table_provider).into();++ Ok(table_provider)+ }++ fn call(&self, args: &[datafusion_expr::Expr]) -> Result<Arc<dyn TableProvider>> {let codec: Arc<dyn LogicalExtensionCodec> = (&self.0.logical_codec).into();let expr_list = LogicalExprList {expr: serialize_exprs(args, codec.as_ref())?,};let filters_serialized = expr_list.encode_to_vec().into();+ #[expect(deprecated)]let table_provider = unsafe { (self.0.call)(&self.0, filters_serialized) };let table_provider = df_result!(table_provider)?;@@ -242,7 +321,9 @@ mod tests {use datafusion::logical_expr::ptr_eq::arc_ptr_eq;use datafusion::prelude::{SessionContext, lit};use datafusion::scalar::ScalarValue;+ use datafusion_catalog::TableFunctionArgs;use datafusion_execution::TaskContextProvider;+ use datafusion_expr::Expr;use super::*;@@ -250,8 +331,12 @@ mod tests {struct TestUDTF {}impl TableFunctionImpl for TestUDTF {- fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(+ &self,+ args: TableFunctionArgs,+ ) -> Result<Arc<dyn TableProvider>> {let args = args+ .exprs().iter().map(|arg| {if let Expr::Literal(scalar, _) = arg {@@ -341,7 +426,10 @@ mod tests {let foreign_udf: Arc<dyn TableFunctionImpl> = local_udtf.into();- let table = foreign_udf.call(&[lit(6_u64), lit("one"), lit(2.0), lit(3_u64)])?;+ let table = foreign_udf.call_with_args(TableFunctionArgs::new(+ &[lit(6_u64), lit("one"), lit(2.0), lit(3_u64)],+ &ctx.state(),+ ))?;let _ = ctx.register_table("test-table", table)?;diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rsindex 342269fbc2..8c2403c3a2 100644--- a/datafusion/functions-table/src/generate_series.rs+++ b/datafusion/functions-table/src/generate_series.rs@@ -23,9 +23,9 @@ use arrow::datatypes::{};use arrow::record_batch::RecordBatch;use async_trait::async_trait;-use datafusion_catalog::Session;use datafusion_catalog::TableFunctionImpl;use datafusion_catalog::TableProvider;+use datafusion_catalog::{Session, TableFunctionArgs};use datafusion_common::{Result, ScalarValue, plan_err};use datafusion_expr::{Expr, TableType};use datafusion_physical_plan::ExecutionPlan;@@ -479,7 +479,8 @@ struct GenerateSeriesFuncImpl {}impl TableFunctionImpl for GenerateSeriesFuncImpl {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();if exprs.is_empty() || exprs.len() > 3 {return plan_err!("{} function requires 1 to 3 arguments", self.name);}@@ -737,12 +738,12 @@ impl GenerateSeriesFuncImpl {pub struct GenerateSeriesFunc {}impl TableFunctionImpl for GenerateSeriesFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {let impl_func = GenerateSeriesFuncImpl {name: "generate_series",include_end: true,};- impl_func.call(exprs)+ impl_func.call_with_args(args)}}@@ -750,12 +751,12 @@ impl TableFunctionImpl for GenerateSeriesFunc {pub struct RangeFunc {}impl TableFunctionImpl for RangeFunc {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {let impl_func = GenerateSeriesFuncImpl {name: "range",include_end: false,};- impl_func.call(exprs)+ impl_func.call_with_args(args)}}diff --git a/docs/source/library-user-guide/functions/adding-udfs.md b/docs/source/library-user-guide/functions/adding-udfs.mdindex 15db731ca3..0221e2e5ad 100644--- a/docs/source/library-user-guide/functions/adding-udfs.md+++ b/docs/source/library-user-guide/functions/adding-udfs.md@@ -1377,15 +1377,16 @@ in the CLI to read the metadata from a Parquet file.The simple UDTF used here takes a single `Int64` argument and returns a table with a single column with the value of theargument. To create a function in DataFusion, you need to implement the `TableFunctionImpl` trait. This trait has a-single method, `call`, that takes a slice of `Expr`s and returns a `Result<Arc<dyn TableProvider>>`.+single method, `call_with_args`, that takes a `TableFunctionArgs` struct and returns a `Result<Arc<dyn TableProvider>>`.+Passed struct includes function arguments as a slice of `Expr`s.-In the `call` method, you parse the input `Expr`s and return a `TableProvider`. You might also want to do some+In the `call_with_args` method, you parse the input `Expr`s and return a `TableProvider`. You might also want to do somevalidation of the input `Expr`s, e.g. checking that the number of arguments is correct.```rustuse std::sync::Arc;use datafusion::common::{plan_err, ScalarValue, Result};-use datafusion::catalog::{TableFunctionImpl, TableProvider};+use datafusion::catalog::{TableFunctionArgs, TableFunctionImpl, TableProvider};use datafusion::arrow::array::{ArrayRef, Int64Array};use datafusion::datasource::memory::MemTable;use arrow::record_batch::RecordBatch;@@ -1397,7 +1398,8 @@ use datafusion_expr::Expr;pub struct EchoFunction {}impl TableFunctionImpl for EchoFunction {- fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+ fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+ let exprs = args.exprs();let Some(Expr::Literal(ScalarValue::Int64(Some(value)), _)) = exprs.get(0) else {return plan_err!("First argument must be an integer");};@@ -1426,7 +1428,7 @@ With the UDTF implemented, you can register it with the `SessionContext`:```rust# use std::sync::Arc;# use datafusion::common::{plan_err, ScalarValue, Result};-# use datafusion::catalog::{TableFunctionImpl, TableProvider};+# use datafusion::catalog::{TableFunctionArgs, TableFunctionImpl, TableProvider};# use datafusion::arrow::array::{ArrayRef, Int64Array};# use datafusion::datasource::memory::MemTable;# use arrow::record_batch::RecordBatch;@@ -1438,7 +1440,8 @@ With the UDTF implemented, you can register it with the `SessionContext`:# pub struct EchoFunction {}## impl TableFunctionImpl for EchoFunction {-# fn call(&self, exprs: &[Expr]) -> Result<Arc<dyn TableProvider>> {+# fn call_with_args(&self, args: TableFunctionArgs) -> Result<Arc<dyn TableProvider>> {+# let exprs = args.exprs();# let Some(Expr::Literal(ScalarValue::Int64(Some(value)), _)) = exprs.get(0) else {# return plan_err!("First argument must be an integer");# };