Apache DataFusion SQL Query Engine
ci: Ensure only the main datafusion crate is used in the examples (#25777)
Nuno Faria991fd23dd0be
diff --git a/.asf.yaml b/.asf.yamlindex f235c6dd97..1307c8759b 100644--- a/.asf.yaml+++ b/.asf.yaml@@ -87,6 +87,7 @@ github:- "check Cargo.toml formatting"- "check configs.md and ***_functions.md is up-to-date"- "check example README is up-to-date"+ - "Check datafusion crates used in datafusion-examples"- "Verify MSRV (Min Supported Rust Version)"# needs to be updated as part of the release process# .asf.yaml doesn't support wildcard branch protection rules, only exact branch namesdiff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex d9f327b401..da702fd633 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -897,6 +897,16 @@ jobs:run: |bash ci/scripts/check_examples_docs.sh+ # Ensures that the examples only use the main `datafusion` crate and not its subcrates, which are+ # already exported by `datafusion` (there are a few exceptions, like `datafusion-proto`).+ # See: https://github.com/apache/datafusion/pull/14305+ examples-check-datafusion-crates:+ name: Check datafusion crates used in datafusion-examples+ runs-on: ubuntu-slim+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - run: python3 ci/scripts/check_examples_datafusion_crates.py+# Verify MSRV for the crates which are directly used by other projects:# - datafusion# - datafusion-substraitdiff --git a/Cargo.lock b/Cargo.lockindex 6573ef5aa7..e76b39aa19 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2139,11 +2139,7 @@ dependencies = ["bytes","dashmap","datafusion",- "datafusion-common",- "datafusion-expr",- "datafusion-physical-expr-adapter","datafusion-proto",- "datafusion-sql","env_logger","futures","insta",diff --git a/ci/scripts/check_examples_datafusion_crates.py b/ci/scripts/check_examples_datafusion_crates.pynew file mode 100644index 0000000000..1a04aef3f4--- /dev/null+++ b/ci/scripts/check_examples_datafusion_crates.py@@ -0,0 +1,54 @@+#!/usr/bin/env python3+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Checks if datafusion-examples uses `datafusion-...` subcrates unnecessarily, as they are already+# exported by the `datafusion` crate.++import tomllib+import sys++allowed_datafusion_crates = {+ "datafusion",+ "datafusion-proto",+ "datafusion-substrait",+}++crates = set()+with open("datafusion-examples/Cargo.toml", "rb") as f:+ cargo = tomllib.load(f)+ for section in ("dependencies", "dev-dependencies", "build-dependencies"):+ crates.update(+ value.get("package", name) if isinstance(value, dict) else name+ for name, value in cargo.get(section, {}).items()+ )++crates = [+ crate for crate in crates if "datafusion" in crate and crate not in allowed_datafusion_crates+]++if len(crates) > 0:+ print(+ "datafusion-examples should only use the main datafusion crate and not its subcrates.",+ file=sys.stderr,+ )+ print(+ f"Please remove the following crates and use `datafusion::...` instead: {crates}.",+ file=sys.stderr,+ )++ sys.exit(1)diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.tomlindex 6d6d917ac4..9b254969fa 100644--- a/datafusion-examples/Cargo.toml+++ b/datafusion-examples/Cargo.toml@@ -38,8 +38,8 @@ workspace = true[dependencies]arrow = { workspace = true }arrow-schema = { workspace = true }+# note: only use the main datafusion crate for examplesdatafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }-datafusion-common = { workspace = true }nom = "8.0.0"tempfile = { workspace = true }tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }@@ -47,14 +47,10 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"][dev-dependencies]arrow-flight = { workspace = true }async-trait = { workspace = true }+base64 = "0.23.0"bytes = { workspace = true }dashmap = { workspace = true }-# note only use main datafusion crate for examples-base64 = "0.23.0"-datafusion-expr = { workspace = true }-datafusion-physical-expr-adapter = { workspace = true }datafusion-proto = { workspace = true, features = ["parquet"] }-datafusion-sql = { workspace = true }env_logger = { workspace = true }futures = { workspace = true }insta = { workspace = true }diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rsindex d8dba65929..a4e5b06ec1 100644--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs@@ -50,11 +50,11 @@ use datafusion::execution::context::SessionContext;use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::parquet::arrow::ArrowWriter;use datafusion::physical_expr::PhysicalExpr;-use datafusion::physical_plan::ExecutionPlan;-use datafusion::prelude::SessionConfig;-use datafusion_physical_expr_adapter::{+use datafusion::physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,};+use datafusion::physical_plan::ExecutionPlan;+use datafusion::prelude::SessionConfig;use datafusion_proto::bytes::{physical_plan_from_bytes_with_proto_converter,physical_plan_to_bytes_with_proto_converter,diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rsindex 202c0a7125..a8b5e6aa10 100644--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs@@ -34,10 +34,10 @@ use datafusion::execution::object_store::ObjectStoreUrl;use datafusion::parquet::arrow::ArrowWriter;use datafusion::physical_expr::PhysicalExpr;use datafusion::physical_expr::expressions::CastExpr;-use datafusion::prelude::SessionConfig;-use datafusion_physical_expr_adapter::{+use datafusion::physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,};+use datafusion::prelude::SessionConfig;use object_store::memory::InMemory;use object_store::path::Path;use object_store::{ObjectStore, ObjectStoreExt, PutPayload};diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rsindex d2024621aa..e39c2047d2 100644--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs@@ -38,12 +38,12 @@ use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::properties::WriterProperties;use datafusion::physical_expr::PhysicalExpr;-use datafusion::physical_plan::ExecutionPlan;-use datafusion::prelude::{SessionConfig, lit};-use datafusion_physical_expr_adapter::{+use datafusion::physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,replace_columns_with_literals,};+use datafusion::physical_plan::ExecutionPlan;+use datafusion::prelude::{SessionConfig, lit};use futures::StreamExt;use object_store::memory::InMemory;use object_store::path::Path;diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rsindex 79023d5581..1e1362954e 100644--- a/datafusion-examples/examples/data_io/json_shredding.rs+++ b/datafusion-examples/examples/data_io/json_shredding.rs@@ -39,11 +39,11 @@ use datafusion::parquet::arrow::ArrowWriter;use datafusion::parquet::file::properties::WriterProperties;use datafusion::physical_expr::PhysicalExpr;use datafusion::physical_expr::{ScalarFunctionExpr, expressions};-use datafusion::prelude::SessionConfig;-use datafusion::scalar::ScalarValue;-use datafusion_physical_expr_adapter::{+use datafusion::physical_expr_adapter::{DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,};+use datafusion::prelude::SessionConfig;+use datafusion::scalar::ScalarValue;use object_store::memory::InMemory;use object_store::path::Path;use object_store::{ObjectStoreExt, PutPayload};diff --git a/datafusion-examples/examples/dataframe/cache_factory.rs b/datafusion-examples/examples/dataframe/cache_factory.rsindex ffbce298b4..4dd9113177 100644--- a/datafusion-examples/examples/dataframe/cache_factory.rs+++ b/datafusion-examples/examples/dataframe/cache_factory.rs@@ -26,6 +26,7 @@ use async_trait::async_trait;use datafusion::catalog::Session;use datafusion::catalog::memory::MemorySourceConfig;use datafusion::common::DFSchemaRef;+use datafusion::common::HashMap;use datafusion::error::Result;use datafusion::execution::context::QueryPlanner;use datafusion::execution::session_state::CacheFactory;@@ -39,7 +40,6 @@ use datafusion::physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner,};use datafusion::prelude::*;-use datafusion_common::HashMap;use datafusion_examples::utils::{datasets::ExampleDataset, write_csv_to_parquet};/// This example demonstrates how to leverage [CacheFactory] to implement custom caching strategies for dataframes in DataFusion.diff --git a/datafusion-examples/examples/extension_types/temperature.rs b/datafusion-examples/examples/extension_types/temperature.rsindex 478cf5ebbf..15167aa6c5 100644--- a/datafusion-examples/examples/extension_types/temperature.rs+++ b/datafusion-examples/examples/extension_types/temperature.rs@@ -23,15 +23,15 @@ use arrow::datatypes::{Float32Type, Float64Type};use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};+use datafusion::common::internal_err;+use datafusion::common::types::DFExtensionType;use datafusion::dataframe::DataFrame;use datafusion::error::Result;use datafusion::execution::SessionStateBuilder;-use datafusion::prelude::SessionContext;-use datafusion_common::internal_err;-use datafusion_common::types::DFExtensionType;-use datafusion_expr::registry::{+use datafusion::logical_expr::registry::{ExtensionTypeRegistration, ExtensionTypeRegistry, MemoryExtensionTypeRegistry,};+use datafusion::prelude::SessionContext;use std::collections::HashMap;use std::fmt::{Display, Write};use std::sync::Arc;diff --git a/datafusion-examples/examples/relation_planner/match_recognize.rs b/datafusion-examples/examples/relation_planner/match_recognize.rsindex c4b3d522ef..795b6eea2b 100644--- a/datafusion-examples/examples/relation_planner/match_recognize.rs+++ b/datafusion-examples/examples/relation_planner/match_recognize.rs@@ -74,16 +74,16 @@ use std::{any::Any, cmp::Ordering, hash::Hasher, sync::Arc};use arrow::array::{ArrayRef, Float64Array, Int32Array, StringArray};use arrow::record_batch::RecordBatch;-use datafusion::prelude::*;-use datafusion_common::{DFSchemaRef, Result};-use datafusion_expr::{+use datafusion::common::{DFSchemaRef, Result};+use datafusion::logical_expr::{Expr, UserDefinedLogicalNode,logical_plan::{Extension, InvariantLevel, LogicalPlan},planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,},};-use datafusion_sql::sqlparser::ast::TableFactor;+use datafusion::prelude::*;+use datafusion::sql::sqlparser::ast::TableFactor;use insta::assert_snapshot;// ============================================================================@@ -281,14 +281,14 @@ impl UserDefinedLogicalNode for MiniMatchRecognizeNode {) -> Result<Arc<dyn UserDefinedLogicalNode>> {let expected_len = self.measures.len() + self.definitions.len();if exprs.len() != expected_len {- return Err(datafusion_common::plan_datafusion_err!(+ return Err(datafusion::common::plan_datafusion_err!("MiniMatchRecognize: expected {expected_len} expressions, got {}",exprs.len()));}let input = inputs.into_iter().next().ok_or_else(|| {- datafusion_common::plan_datafusion_err!(+ datafusion::common::plan_datafusion_err!("MiniMatchRecognize requires exactly one input")})?;diff --git a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs b/datafusion-examples/examples/relation_planner/pivot_unpivot.rsindex 4b721346aa..01293fe3d6 100644--- a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs+++ b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs@@ -64,16 +64,16 @@ use std::sync::Arc;use arrow::array::{ArrayRef, Int64Array, StringArray};use arrow::record_batch::RecordBatch;-use datafusion::prelude::*;-use datafusion_common::{Result, ScalarValue, plan_datafusion_err};-use datafusion_expr::{+use datafusion::common::{Result, ScalarValue, plan_datafusion_err};+use datafusion::logical_expr::{Expr, case, col, lit,logical_plan::builder::LogicalPlanBuilder,planner::{PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,},};-use datafusion_sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};+use datafusion::prelude::*;+use datafusion::sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};use insta::assert_snapshot;// ============================================================================@@ -399,10 +399,10 @@ impl RelationPlanner for PivotUnpivotPlanner {fn plan_pivot(ctx: &mut dyn RelationPlannerContext,table: TableFactor,- aggregate_functions: &[datafusion_sql::sqlparser::ast::ExprWithAlias],- value_column: &[datafusion_sql::sqlparser::ast::Expr],+ aggregate_functions: &[datafusion::sql::sqlparser::ast::ExprWithAlias],+ value_column: &[datafusion::sql::sqlparser::ast::Expr],value_source: PivotValueSource,- alias: Option<datafusion_sql::sqlparser::ast::TableAlias>,+ alias: Option<datafusion::sql::sqlparser::ast::TableAlias>,) -> Result<RelationPlanning> {// Plan the input tablelet input = ctx.plan(table)?;@@ -520,11 +520,11 @@ fn plan_pivot(fn plan_unpivot(ctx: &mut dyn RelationPlannerContext,table: TableFactor,- value: &datafusion_sql::sqlparser::ast::Expr,- name: datafusion_sql::sqlparser::ast::Ident,- columns: &[datafusion_sql::sqlparser::ast::ExprWithAlias],+ value: &datafusion::sql::sqlparser::ast::Expr,+ name: datafusion::sql::sqlparser::ast::Ident,+ columns: &[datafusion::sql::sqlparser::ast::ExprWithAlias],null_inclusion: Option<&NullInclusion>,- alias: Option<datafusion_sql::sqlparser::ast::TableAlias>,+ alias: Option<datafusion::sql::sqlparser::ast::TableAlias>,) -> Result<RelationPlanning> {// Plan the input tablelet input = ctx.plan(table)?;diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex 7a8f533ac3..9a411080a7 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -100,6 +100,21 @@ use futures::{use rand::{Rng, SeedableRng, rngs::StdRng};use tonic::async_trait;+use datafusion::common::{+ DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,+};+use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;+use datafusion::logical_expr::{+ UserDefinedLogicalNode, UserDefinedLogicalNodeCore,+ logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},+ planner::{+ PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,+ },+};+use datafusion::sql::sqlparser::ast::{+ self, TableFactor, TableSampleMethod, TableSampleUnit,+};use datafusion::{catalog::Session,execution::{@@ -119,21 +134,6 @@ use datafusion::{optimizer::simplify_expressions::simplify_literal::parse_literal,physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions},};-use datafusion_common::{- DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,- plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,-};-use datafusion_expr::physical_planning_context::PhysicalPlanningContext;-use datafusion_expr::{- UserDefinedLogicalNode, UserDefinedLogicalNodeCore,- logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},- planner::{- PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,- },-};-use datafusion_sql::sqlparser::ast::{- self, TableFactor, TableSampleMethod, TableSampleUnit,-};use insta::assert_snapshot;// ============================================================================diff --git a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs b/datafusion-examples/examples/sql_ops/custom_sql_parser.rsindex ccf511917a..465c18090f 100644--- a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs+++ b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs@@ -44,8 +44,10 @@ use datafusion::catalog::{CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider,TableProviderFactory,};+use datafusion::common::{DFSchema, TableReference, plan_datafusion_err, plan_err};use datafusion::datasource::listing_table_factory::ListingTableFactory;use datafusion::error::{DataFusionError, Result};+use datafusion::logical_expr::CreateExternalTable;use datafusion::prelude::SessionContext;use datafusion::sql::{parser::{DFParser, DFParserBuilder, Statement},@@ -55,8 +57,6 @@ use datafusion::sql::{tokenizer::Token,},};-use datafusion_common::{DFSchema, TableReference, plan_datafusion_err, plan_err};-use datafusion_expr::CreateExternalTable;use futures::StreamExt;use insta::assert_snapshot;use object_store::ObjectStore;diff --git a/datafusion-examples/examples/udf/table_list_udtf.rs b/datafusion-examples/examples/udf/table_list_udtf.rsindex 739f8e11f0..420634852e 100644--- a/datafusion-examples/examples/udf/table_list_udtf.rs+++ b/datafusion-examples/examples/udf/table_list_udtf.rs@@ -21,13 +21,13 @@ use std::sync::{Arc, LazyLock};use arrow::array::{RecordBatch, StringBuilder};use arrow_schema::{DataType, Field, Schema, SchemaRef};+use datafusion::common::{DataFusionError, plan_err};use datafusion::{catalog::{MemTable, TableFunctionArgs, TableFunctionImpl, TableProvider},common::Result,execution::SessionState,prelude::SessionContext,};-use datafusion_common::{DataFusionError, plan_err};use tokio::{runtime::Handle, task::block_in_place};const FUNCTION_NAME: &str = "table_list";