Skip to content

Apache DataFusion SQL Query Engine

ci: Ensure only the main datafusion crate is used in the examples (#25777)

Nuno Faria991fd23dd0be
diff --git a/.asf.yaml b/.asf.yaml
index f235c6dd97..1307c8759b 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -87,6 +87,7 @@ github:
- "check Cargo.toml formatting"
- "check configs.md and ***_functions.md is up-to-date"
- "check example README is up-to-date"
+ - "Check datafusion crates used in datafusion-examples"
- "Verify MSRV (Min Supported Rust Version)"
# needs to be updated as part of the release process
# .asf.yaml doesn't support wildcard branch protection rules, only exact branch names
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index d9f327b401..da702fd633 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -897,6 +897,16 @@ jobs:
run: |
bash ci/scripts/check_examples_docs.sh
+ # Ensures that the examples only use the main `datafusion` crate and not its subcrates, which are
+ # already exported by `datafusion` (there are a few exceptions, like `datafusion-proto`).
+ # See: https://github.com/apache/datafusion/pull/14305
+ examples-check-datafusion-crates:
+ name: Check datafusion crates used in datafusion-examples
+ runs-on: ubuntu-slim
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - run: python3 ci/scripts/check_examples_datafusion_crates.py
+
# Verify MSRV for the crates which are directly used by other projects:
# - datafusion
# - datafusion-substrait
diff --git a/Cargo.lock b/Cargo.lock
index 6573ef5aa7..e76b39aa19 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2139,11 +2139,7 @@ dependencies = [
"bytes",
"dashmap",
"datafusion",
- "datafusion-common",
- "datafusion-expr",
- "datafusion-physical-expr-adapter",
"datafusion-proto",
- "datafusion-sql",
"env_logger",
"futures",
"insta",
diff --git a/ci/scripts/check_examples_datafusion_crates.py b/ci/scripts/check_examples_datafusion_crates.py
new file mode 100644
index 0000000000..1a04aef3f4
--- /dev/null
+++ b/ci/scripts/check_examples_datafusion_crates.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Checks if datafusion-examples uses `datafusion-...` subcrates unnecessarily, as they are already
+# exported by the `datafusion` crate.
+
+import tomllib
+import sys
+
+allowed_datafusion_crates = {
+ "datafusion",
+ "datafusion-proto",
+ "datafusion-substrait",
+}
+
+crates = set()
+with open("datafusion-examples/Cargo.toml", "rb") as f:
+ cargo = tomllib.load(f)
+ for section in ("dependencies", "dev-dependencies", "build-dependencies"):
+ crates.update(
+ value.get("package", name) if isinstance(value, dict) else name
+ for name, value in cargo.get(section, {}).items()
+ )
+
+crates = [
+ crate for crate in crates if "datafusion" in crate and crate not in allowed_datafusion_crates
+]
+
+if len(crates) > 0:
+ print(
+ "datafusion-examples should only use the main datafusion crate and not its subcrates.",
+ file=sys.stderr,
+ )
+ print(
+ f"Please remove the following crates and use `datafusion::...` instead: {crates}.",
+ file=sys.stderr,
+ )
+
+ sys.exit(1)
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 6d6d917ac4..9b254969fa 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -38,8 +38,8 @@ workspace = true
[dependencies]
arrow = { workspace = true }
arrow-schema = { workspace = true }
+# note: only use the main datafusion crate for examples
datafusion = { workspace = true, default-features = true, features = ["parquet_encryption"] }
-datafusion-common = { workspace = true }
nom = "8.0.0"
tempfile = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] }
@@ -47,14 +47,10 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"]
[dev-dependencies]
arrow-flight = { workspace = true }
async-trait = { workspace = true }
+base64 = "0.23.0"
bytes = { workspace = true }
dashmap = { workspace = true }
-# note only use main datafusion crate for examples
-base64 = "0.23.0"
-datafusion-expr = { workspace = true }
-datafusion-physical-expr-adapter = { workspace = true }
datafusion-proto = { workspace = true, features = ["parquet"] }
-datafusion-sql = { workspace = true }
env_logger = { workspace = true }
futures = { workspace = true }
insta = { workspace = true }
diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
index d8dba65929..a4e5b06ec1 100644
--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
@@ -50,11 +50,11 @@ use datafusion::execution::context::SessionContext;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::SessionConfig;
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
};
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::SessionConfig;
use datafusion_proto::bytes::{
physical_plan_from_bytes_with_proto_converter,
physical_plan_to_bytes_with_proto_converter,
diff --git a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
index 202c0a7125..a8b5e6aa10 100644
--- a/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_file_casts.rs
@@ -34,10 +34,10 @@ use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_expr::expressions::CastExpr;
-use datafusion::prelude::SessionConfig;
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
};
+use datafusion::prelude::SessionConfig;
use object_store::memory::InMemory;
use object_store::path::Path;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rs
index d2024621aa..e39c2047d2 100644
--- a/datafusion-examples/examples/custom_data_source/default_column_values.rs
+++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs
@@ -38,12 +38,12 @@ use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType};
use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::properties::WriterProperties;
use datafusion::physical_expr::PhysicalExpr;
-use datafusion::physical_plan::ExecutionPlan;
-use datafusion::prelude::{SessionConfig, lit};
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
replace_columns_with_literals,
};
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion::prelude::{SessionConfig, lit};
use futures::StreamExt;
use object_store::memory::InMemory;
use object_store::path::Path;
diff --git a/datafusion-examples/examples/data_io/json_shredding.rs b/datafusion-examples/examples/data_io/json_shredding.rs
index 79023d5581..1e1362954e 100644
--- a/datafusion-examples/examples/data_io/json_shredding.rs
+++ b/datafusion-examples/examples/data_io/json_shredding.rs
@@ -39,11 +39,11 @@ use datafusion::parquet::arrow::ArrowWriter;
use datafusion::parquet::file::properties::WriterProperties;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_expr::{ScalarFunctionExpr, expressions};
-use datafusion::prelude::SessionConfig;
-use datafusion::scalar::ScalarValue;
-use datafusion_physical_expr_adapter::{
+use datafusion::physical_expr_adapter::{
DefaultPhysicalExprAdapterFactory, PhysicalExprAdapter, PhysicalExprAdapterFactory,
};
+use datafusion::prelude::SessionConfig;
+use datafusion::scalar::ScalarValue;
use object_store::memory::InMemory;
use object_store::path::Path;
use object_store::{ObjectStoreExt, PutPayload};
diff --git a/datafusion-examples/examples/dataframe/cache_factory.rs b/datafusion-examples/examples/dataframe/cache_factory.rs
index ffbce298b4..4dd9113177 100644
--- a/datafusion-examples/examples/dataframe/cache_factory.rs
+++ b/datafusion-examples/examples/dataframe/cache_factory.rs
@@ -26,6 +26,7 @@ use async_trait::async_trait;
use datafusion::catalog::Session;
use datafusion::catalog::memory::MemorySourceConfig;
use datafusion::common::DFSchemaRef;
+use datafusion::common::HashMap;
use datafusion::error::Result;
use datafusion::execution::context::QueryPlanner;
use datafusion::execution::session_state::CacheFactory;
@@ -39,7 +40,6 @@ use datafusion::physical_planner::{
DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner,
};
use datafusion::prelude::*;
-use datafusion_common::HashMap;
use datafusion_examples::utils::{datasets::ExampleDataset, write_csv_to_parquet};
/// This example demonstrates how to leverage [CacheFactory] to implement custom caching strategies for dataframes in DataFusion.
diff --git a/datafusion-examples/examples/extension_types/temperature.rs b/datafusion-examples/examples/extension_types/temperature.rs
index 478cf5ebbf..15167aa6c5 100644
--- a/datafusion-examples/examples/extension_types/temperature.rs
+++ b/datafusion-examples/examples/extension_types/temperature.rs
@@ -23,15 +23,15 @@ use arrow::datatypes::{Float32Type, Float64Type};
use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
+use datafusion::common::internal_err;
+use datafusion::common::types::DFExtensionType;
use datafusion::dataframe::DataFrame;
use datafusion::error::Result;
use datafusion::execution::SessionStateBuilder;
-use datafusion::prelude::SessionContext;
-use datafusion_common::internal_err;
-use datafusion_common::types::DFExtensionType;
-use datafusion_expr::registry::{
+use datafusion::logical_expr::registry::{
ExtensionTypeRegistration, ExtensionTypeRegistry, MemoryExtensionTypeRegistry,
};
+use datafusion::prelude::SessionContext;
use std::collections::HashMap;
use std::fmt::{Display, Write};
use std::sync::Arc;
diff --git a/datafusion-examples/examples/relation_planner/match_recognize.rs b/datafusion-examples/examples/relation_planner/match_recognize.rs
index c4b3d522ef..795b6eea2b 100644
--- a/datafusion-examples/examples/relation_planner/match_recognize.rs
+++ b/datafusion-examples/examples/relation_planner/match_recognize.rs
@@ -74,16 +74,16 @@ use std::{any::Any, cmp::Ordering, hash::Hasher, sync::Arc};
use arrow::array::{ArrayRef, Float64Array, Int32Array, StringArray};
use arrow::record_batch::RecordBatch;
-use datafusion::prelude::*;
-use datafusion_common::{DFSchemaRef, Result};
-use datafusion_expr::{
+use datafusion::common::{DFSchemaRef, Result};
+use datafusion::logical_expr::{
Expr, UserDefinedLogicalNode,
logical_plan::{Extension, InvariantLevel, LogicalPlan},
planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
},
};
-use datafusion_sql::sqlparser::ast::TableFactor;
+use datafusion::prelude::*;
+use datafusion::sql::sqlparser::ast::TableFactor;
use insta::assert_snapshot;
// ============================================================================
@@ -281,14 +281,14 @@ impl UserDefinedLogicalNode for MiniMatchRecognizeNode {
) -> Result<Arc<dyn UserDefinedLogicalNode>> {
let expected_len = self.measures.len() + self.definitions.len();
if exprs.len() != expected_len {
- return Err(datafusion_common::plan_datafusion_err!(
+ return Err(datafusion::common::plan_datafusion_err!(
"MiniMatchRecognize: expected {expected_len} expressions, got {}",
exprs.len()
));
}
let input = inputs.into_iter().next().ok_or_else(|| {
- datafusion_common::plan_datafusion_err!(
+ datafusion::common::plan_datafusion_err!(
"MiniMatchRecognize requires exactly one input"
)
})?;
diff --git a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
index 4b721346aa..01293fe3d6 100644
--- a/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
+++ b/datafusion-examples/examples/relation_planner/pivot_unpivot.rs
@@ -64,16 +64,16 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, Int64Array, StringArray};
use arrow::record_batch::RecordBatch;
-use datafusion::prelude::*;
-use datafusion_common::{Result, ScalarValue, plan_datafusion_err};
-use datafusion_expr::{
+use datafusion::common::{Result, ScalarValue, plan_datafusion_err};
+use datafusion::logical_expr::{
Expr, case, col, lit,
logical_plan::builder::LogicalPlanBuilder,
planner::{
PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
},
};
-use datafusion_sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};
+use datafusion::prelude::*;
+use datafusion::sql::sqlparser::ast::{NullInclusion, PivotValueSource, TableFactor};
use insta::assert_snapshot;
// ============================================================================
@@ -399,10 +399,10 @@ impl RelationPlanner for PivotUnpivotPlanner {
fn plan_pivot(
ctx: &mut dyn RelationPlannerContext,
table: TableFactor,
- aggregate_functions: &[datafusion_sql::sqlparser::ast::ExprWithAlias],
- value_column: &[datafusion_sql::sqlparser::ast::Expr],
+ aggregate_functions: &[datafusion::sql::sqlparser::ast::ExprWithAlias],
+ value_column: &[datafusion::sql::sqlparser::ast::Expr],
value_source: PivotValueSource,
- alias: Option<datafusion_sql::sqlparser::ast::TableAlias>,
+ alias: Option<datafusion::sql::sqlparser::ast::TableAlias>,
) -> Result<RelationPlanning> {
// Plan the input table
let input = ctx.plan(table)?;
@@ -520,11 +520,11 @@ fn plan_pivot(
fn plan_unpivot(
ctx: &mut dyn RelationPlannerContext,
table: TableFactor,
- value: &datafusion_sql::sqlparser::ast::Expr,
- name: datafusion_sql::sqlparser::ast::Ident,
- columns: &[datafusion_sql::sqlparser::ast::ExprWithAlias],
+ value: &datafusion::sql::sqlparser::ast::Expr,
+ name: datafusion::sql::sqlparser::ast::Ident,
+ columns: &[datafusion::sql::sqlparser::ast::ExprWithAlias],
null_inclusion: Option<&NullInclusion>,
- alias: Option<datafusion_sql::sqlparser::ast::TableAlias>,
+ alias: Option<datafusion::sql::sqlparser::ast::TableAlias>,
) -> Result<RelationPlanning> {
// Plan the input table
let input = ctx.plan(table)?;
diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs
index 7a8f533ac3..9a411080a7 100644
--- a/datafusion-examples/examples/relation_planner/table_sample.rs
+++ b/datafusion-examples/examples/relation_planner/table_sample.rs
@@ -100,6 +100,21 @@ use futures::{
use rand::{Rng, SeedableRng, rngs::StdRng};
use tonic::async_trait;
+use datafusion::common::{
+ DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,
+};
+use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
+use datafusion::logical_expr::{
+ UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
+ logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},
+ planner::{
+ PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
+ },
+};
+use datafusion::sql::sqlparser::ast::{
+ self, TableFactor, TableSampleMethod, TableSampleUnit,
+};
use datafusion::{
catalog::Session,
execution::{
@@ -119,21 +134,6 @@ use datafusion::{
optimizer::simplify_expressions::simplify_literal::parse_literal,
physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions},
};
-use datafusion_common::{
- DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
- plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,
-};
-use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
-use datafusion_expr::{
- UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
- logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},
- planner::{
- PlannedRelation, RelationPlanner, RelationPlannerContext, RelationPlanning,
- },
-};
-use datafusion_sql::sqlparser::ast::{
- self, TableFactor, TableSampleMethod, TableSampleUnit,
-};
use insta::assert_snapshot;
// ============================================================================
diff --git a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs
index ccf511917a..465c18090f 100644
--- a/datafusion-examples/examples/sql_ops/custom_sql_parser.rs
+++ b/datafusion-examples/examples/sql_ops/custom_sql_parser.rs
@@ -44,8 +44,10 @@ use datafusion::catalog::{
CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider,
TableProviderFactory,
};
+use datafusion::common::{DFSchema, TableReference, plan_datafusion_err, plan_err};
use datafusion::datasource::listing_table_factory::ListingTableFactory;
use datafusion::error::{DataFusionError, Result};
+use datafusion::logical_expr::CreateExternalTable;
use datafusion::prelude::SessionContext;
use datafusion::sql::{
parser::{DFParser, DFParserBuilder, Statement},
@@ -55,8 +57,6 @@ use datafusion::sql::{
tokenizer::Token,
},
};
-use datafusion_common::{DFSchema, TableReference, plan_datafusion_err, plan_err};
-use datafusion_expr::CreateExternalTable;
use futures::StreamExt;
use insta::assert_snapshot;
use object_store::ObjectStore;
diff --git a/datafusion-examples/examples/udf/table_list_udtf.rs b/datafusion-examples/examples/udf/table_list_udtf.rs
index 739f8e11f0..420634852e 100644
--- a/datafusion-examples/examples/udf/table_list_udtf.rs
+++ b/datafusion-examples/examples/udf/table_list_udtf.rs
@@ -21,13 +21,13 @@ use std::sync::{Arc, LazyLock};
use arrow::array::{RecordBatch, StringBuilder};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion::common::{DataFusionError, plan_err};
use datafusion::{
catalog::{MemTable, TableFunctionArgs, TableFunctionImpl, TableProvider},
common::Result,
execution::SessionState,
prelude::SessionContext,
};
-use datafusion_common::{DataFusionError, plan_err};
use tokio::{runtime::Handle, task::block_in_place};
const FUNCTION_NAME: &str = "table_list";