Skip to content

Apache DataFusion SQL Query Engine

16.6 KB
Raw Blame
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

[workspace]
members = [
    "datafusion/common",
    "datafusion/common-runtime",
    "datafusion/catalog",
    "datafusion/catalog-listing",
    "datafusion/datasource",
    "datafusion/datasource-arrow",
    "datafusion/datasource-avro",
    "datafusion/datasource-csv",
    "datafusion/datasource-json",
    "datafusion/datasource-parquet",
    "datafusion/core",
    "datafusion/expr",
    "datafusion/expr-common",
    "datafusion/execution",
    "datafusion/ffi",
    "datafusion/ffi/integration-tests",
    "datafusion/fuzz",
    "datafusion/functions",
    "datafusion/functions-aggregate",
    "datafusion/functions-aggregate-common",
    "datafusion/functions-table",
    "datafusion/functions-nested",
    "datafusion/functions-window",
    "datafusion/functions-window-common",
    "datafusion/optimizer",
    "datafusion/physical-expr",
    "datafusion/physical-expr-adapter",
    "datafusion/physical-expr-common",
    "datafusion/physical-optimizer",
    "datafusion/pruning",
    "datafusion/physical-plan",
    "datafusion/proto",
    "datafusion/proto-common",
    "datafusion/proto-common/gen",
    "datafusion/proto-models",
    "datafusion/proto-models/gen",
    "datafusion/session",
    "datafusion/sql",
    "datafusion/sqllogictest",
    "datafusion/substrait",
    "datafusion/wasmtest",
    "datafusion-cli",
    "datafusion-examples",
    "datafusion-examples/examples/ffi/ffi_example_table_provider",
    "datafusion-examples/examples/ffi/ffi_module_interface",
    "datafusion-examples/examples/ffi/ffi_module_loader",
    "test-utils",
    "benchmarks",
    "datafusion/macros",
    "datafusion/doc",
    "xtask",
]
resolver = "2"

[workspace.package]
authors = ["Apache DataFusion <[email protected]>"]
edition = "2024"
homepage = "https://datafusion.apache.org"
license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/apache/datafusion"
# Define Minimum Supported Rust Version (MSRV)
rust-version = "1.95.0"
# Define DataFusion version
version = "57.0.0-dev"

[workspace.dependencies]
# We turn off default-features for some dependencies here so the workspaces which inherit them can
# selectively turn them on if needed, since we can override default-features = true (from false)
# for the inherited dependency but cannot do the reverse (override from true to false).
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
arrow = { version = "60.1.0", features = [
    "prettyprint",
    "chrono-tz",
] }
arrow-avro = { version = "60.1.0", default-features = false, features = [
    "deflate",
    "snappy",
    "zstd",
    "bzip2",
    "xz",
] }
arrow-buffer = { version = "60.1.0", default-features = false }
arrow-data = { version = "60.1.0", default-features = false }
arrow-flight = { version = "60.1.0", features = [
    "flight-sql",
] }
# Both codecs are required here to make sure that code paths like
# file-spilling have access to all compression codecs.
arrow-ipc = { version = "60.1.0", default-features = false, features = [
    "lz4",
    "zstd",
] }
arrow-ord = { version = "60.1.0", default-features = false }
arrow-schema = { version = "60.1.0", default-features = false }
arrow-select = { version = "60.1.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
bzip2 = "0.6.1"
chrono = { version = "0.4.45", default-features = false }
criterion = "0.8"
ctor = "1.0.7"
dashmap = "6.2.1"
datafusion = { path = "datafusion/core", version = "57.0.0-dev", default-features = false }
datafusion-catalog = { path = "datafusion/catalog", version = "57.0.0-dev" }
datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "57.0.0-dev" }
datafusion-common = { path = "datafusion/common", version = "57.0.0-dev", default-features = false }
datafusion-common-runtime = { path = "datafusion/common-runtime", version = "57.0.0-dev" }
datafusion-datasource = { path = "datafusion/datasource", version = "57.0.0-dev", default-features = false }
datafusion-datasource-arrow = { path = "datafusion/datasource-arrow", version = "57.0.0-dev", default-features = false }
datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "57.0.0-dev", default-features = false }
datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "57.0.0-dev", default-features = false }
datafusion-datasource-json = { path = "datafusion/datasource-json", version = "57.0.0-dev", default-features = false }
datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "57.0.0-dev", default-features = false }
datafusion-doc = { path = "datafusion/doc", version = "57.0.0-dev" }
datafusion-execution = { path = "datafusion/execution", version = "57.0.0-dev", default-features = false }
datafusion-expr = { path = "datafusion/expr", version = "57.0.0-dev", default-features = false }
datafusion-expr-common = { path = "datafusion/expr-common", version = "57.0.0-dev" }
datafusion-ffi = { path = "datafusion/ffi", version = "57.0.0-dev" }
datafusion-functions = { path = "datafusion/functions", version = "57.0.0-dev" }
datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "57.0.0-dev" }
datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "57.0.0-dev" }
datafusion-functions-nested = { path = "datafusion/functions-nested", version = "57.0.0-dev", default-features = false }
datafusion-functions-table = { path = "datafusion/functions-table", version = "57.0.0-dev" }
datafusion-functions-window = { path = "datafusion/functions-window", version = "57.0.0-dev" }
datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "57.0.0-dev" }
datafusion-macros = { path = "datafusion/macros", version = "57.0.0-dev" }
datafusion-optimizer = { path = "datafusion/optimizer", version = "57.0.0-dev", default-features = false }
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "57.0.0-dev", default-features = false }
datafusion-physical-expr-adapter = { path = "datafusion/physical-expr-adapter", version = "57.0.0-dev", default-features = false }
datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "57.0.0-dev", default-features = false }
datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "57.0.0-dev" }
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "57.0.0-dev" }
datafusion-proto = { path = "datafusion/proto", version = "57.0.0-dev", default-features = false }
datafusion-proto-common = { path = "datafusion/proto-common", version = "57.0.0-dev" }
datafusion-proto-models = { path = "datafusion/proto-models", version = "57.0.0-dev" }
datafusion-pruning = { path = "datafusion/pruning", version = "57.0.0-dev" }
datafusion-session = { path = "datafusion/session", version = "57.0.0-dev" }
datafusion-sql = { path = "datafusion/sql", version = "57.0.0-dev" }
datafusion-substrait = { path = "datafusion/substrait", version = "57.0.0-dev" }

doc-comment = "0.3"
env_logger = "0.11"
flate2 = "1.1.9"
futures = "0.3"
glob = "0.3.0"
half = { version = "2.7.0", default-features = false }
hashbrown = { version = "0.17.1" }
hex = { version = "0.4.3" }
indexmap = "2.14.0"
insta = { version = "1.47.2", features = ["glob", "filters"] }
itertools = "0.15"
itoa = "1.0"
liblzma = { version = "0.4.6", features = ["static"] }
log = "^0.4"
loom = "0.7"
memchr = "2.8.1"
num-traits = { version = "0.2" }
object_store = { version = "0.14.2", default-features = false }
parking_lot = "0.13"
parquet = { version = "60.1.0", default-features = false, features = [
    "arrow",
    "async",
    "object_store",
] }
pbjson = { version = "0.9.0" }
pbjson-types = "0.9"
percent-encoding = "2.3"
pin-project = "1"
pin-project-lite = "^0.2.7"
# Should match arrow-flight's version of prost.
prost = "0.14.1"
rand = "0.9"
recursive = "0.1.1"
regex = "1.12"
rstest = "0.28.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "^0.11.0"
sqlparser = { version = "0.63.0", default-features = false, features = ["std", "visitor"] }
stacker = "0.1.24"
strum = "0.28.0"
strum_macros = "0.28.0"
tempfile = "3"
testcontainers-modules = { version = "0.15" }
tokio = { version = "1.52", features = ["macros", "rt", "sync"] }
tokio-stream = "0.1"
tokio-util = "0.7"
url = "2.5.7"
uuid = "1.23"
zstd = { version = "0.14", default-features = false }

# Keep this list sorted alphabetically.
# See https://github.com/apache/datafusion/issues/18467 for the ongoing effort of
# picking useful non-default lints.
[workspace.lints.clippy]
all = { level = "warn", priority = -1 }
pedantic = { level = "warn", priority = -1 }

# https://github.com/apache/datafusion/issues/18881
allow_attributes = "warn"
as_ptr_cast_mut = "warn"
branches_sharing_code = "warn"
clear_with_drain = "warn"
coerce_container_to_any = "warn"
debug_assert_with_mut_call = "warn"
default_union_representation = "warn"
doc_include_without_cfg = "warn"
empty_enum_variants_with_brackets = "warn"
equatable_if_let = "warn"
exit = "warn"
fn_to_numeric_cast_any = "warn"
imprecise_flops = "warn"
infinite_loop = "warn"
iter_on_empty_collections = "warn"
iter_on_single_items = "warn"
large_include_file = "warn"
# Guards against stack overflows
large_stack_frames = "warn"
# Catches `"{foo}"` where the string is never actually formatted
literal_string_with_formatting_args = "warn"
mem_forget = "warn"
needless_type_cast = "warn"
negative_feature_names = "warn"
non_zero_suggestions = "warn"
nonstandard_macro_braces = "warn"
or_fun_call = "warn"
path_buf_push_overwrite = "warn"
pathbuf_init_then_push = "warn"
precedence_bits = "warn"
pub_without_shorthand = "warn"
rc_mutex = "warn"
rest_pat_in_fully_bound_structs = "warn"
# Avoids hashing the key twice
set_contains_or_insert = "warn"
single_option_map = "warn"
string_lit_as_bytes = "warn"
string_lit_chars_any = "warn"
suspicious_xor_used_as_pow = "warn"
trailing_empty_array = "warn"
trait_duplication_in_bounds = "warn"
uninhabited_references = "warn"
unnecessary_safety_doc = "warn"
unnecessary_self_imports = "warn"
unnecessary_struct_initialization = "warn"
unused_peekable = "warn"
unused_rounding = "warn"
verbose_file_reads = "warn"
wildcard_dependencies = "warn"

# Pedantic lints we opt out of, with the number of hits at the time we enabled `pedantic`.
# Some of these we should consider enabling.
assert_is_empty = "allow"                    # 105 hits
cast_lossless = "allow"                      # 361 hits
cast_possible_truncation = "allow"           # 911 hits
cast_possible_wrap = "allow"                 # 493 hits
cast_precision_loss = "allow"                # 266 hits
cast_sign_loss = "allow"                     # 440 hits
default_trait_access = "allow"               # 221 hits
doc_markdown = "allow"                       # 4933 hits; needs a long `doc-valid-idents` list in `clippy.toml`
enum_glob_use = "allow"                      # 98 hits
explicit_into_iter_loop = "allow"            # 55 hits
explicit_iter_loop = "allow"                 # 189 hits
float_cmp = "allow"                          # 8 hits; exact float comparisons are often intentional here
if_not_else = "allow"                        # 133 hits
ignored_unit_patterns = "allow"              # 52 hits
implicit_clone = "allow"                     # 198 hits
implicit_hasher = "allow"                    # 17 hits; some sites feed arrow APIs that require the default hasher
inline_always = "allow"                      # 45 hits
items_after_statements = "allow"             # 171 hits
manual_string_new = "allow"                  # 84 hits
many_single_char_names = "allow"             # 12 hits; short names are idiomatic in the numeric kernels
map_unwrap_or = "allow"                      # 198 hits
match_bool = "allow"                         # 46 hits
match_same_arms = "allow"                    # 261 hits
match_wildcard_for_single_variants = "allow" # 132 hits
missing_errors_doc = "allow"                 # 1807 hits
missing_fields_in_debug = "allow"            # 29 hits
missing_panics_doc = "allow"                 # 244 hits
must_use_candidate = "allow"                 # 2726 hits
needless_raw_string_hashes = "allow"         # 540 hits
redundant_closure_for_method_calls = "allow" # 686 hits
redundant_else = "allow"                     # 48 hits
return_self_not_must_use = "allow"           # 644 hits
semicolon_if_nothing_returned = "allow"      # 1353 hits
similar_names = "allow"                      # 228 hits; too many false positives, e.g. `expr`/`exprs`
single_match_else = "allow"                  # 155 hits
struct_excessive_bools = "allow"             # 24 hits
struct_field_names = "allow"                 # 14 hits
too_many_lines = "allow"                     # 484 hits
trivially_copy_pass_by_ref = "allow"         # 74 hits
unnecessary_literal_bound = "allow"          # 471 hits
unnecessary_wraps = "allow"                  # 427 hits
unnested_or_patterns = "allow"               # 68 hits
unreadable_literal = "allow"                 # 502 hits
unused_self = "allow"                        # 69 hits
used_underscore_items = "allow"              # 28 hits
wildcard_imports = "allow"                   # 48 hits; `use crate::prelude::*` is idiomatic

[workspace.lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = [
    'cfg(datafusion_coop, values("tokio", "tokio_fallback", "per_stream"))',
    "cfg(coverage)",
    "cfg(coverage_nightly)",
    # Enables the loom model-checked concurrency tests; see
    # datafusion/physical-plan/src/joins/hash_join/probe_completion.rs.
    # Namespaced rather than a bare `loom`, which tokio also reads and which
    # would switch tokio itself into its loom build.
    "cfg(datafusion_loom)",
] }
unused_qualifications = "deny"

# --------------------
# Compilation Profiles
# --------------------
# A Cargo profile is a preset for the compiler/linker knobs that trades off:
# - Build time: how quickly code compiles and links
# - Runtime performance: how fast the resulting binaries execute
# - Binary size: how large the executables end up
# - Debuggability: how much debug information is preserved for debugging and profiling
#
# To use a profile: `cargo [ build | run | ... ] --profile <profile-name>`
#
# Profiles available:
# - dev: default debug build; fastest to compile, slowest to run. Carries line
#     tables only, so panics and backtraces still name a file and line but a
#     debugger cannot inspect variables; use the `dbg` profile when you need
#     that. For everyday development; default for "cargo [ build | test | run ]".
# - dbg: like dev, but with full debug info for stepping through code and
#     inspecting variables in a debugger.
# - release: fully optimized build; slowest to compile, fastest to run, smallest
#     binaries. For public releases; default for "cargo [ bench | install ]".
# - release-nonlto: skips LTO, so it builds much faster while staying close to
#     release performance. Useful when developing performance optimizations.
# - profiling: inherits release optimizations but retains debug info to support
#     profiling tools and flamegraphs.
# - ci: derived from `dev` but disables debug info and incremental builds to keep
#     CI artifacts small and reproducible.
# - ci-optimized: derived from `release` but enables debug assertions and uses
#     less aggressive optimizations for faster builds.  Used for long-running CI
#     tasks.
#
# If you want to optimize compilation, the `compile_profile` benchmark can be useful.
[profile.dev]
debug = "line-tables-only"

# Full debug info for interactive debugging ("debug" is a reserved profile name).
[profile.dbg]
inherits = "dev"
debug = true

[profile.release]
codegen-units = 1
lto = true
strip = true      # Eliminate debug info to minimize binary size

[profile.release-nonlto]
inherits = "release"
codegen-units = 16
lto = false
strip = false        # Retain debug info for flamegraphs

[profile.profiling]
inherits = "release"
debug = true
strip = false

[profile.ci-optimized]
inherits = "release"
debug-assertions = true
codegen-units = 16
lto = "thin"

[profile.ci]
inherits = "dev"
debug = false
incremental = false

# This rule applies to every package except workspace members (dependencies
# such as `arrow` and `tokio`). It disables debug info and related features on
# dependencies so their binaries stay smaller, improving cache reuse.
[profile.ci.package."*"]
debug = false
debug-assertions = false
strip = "debuginfo"
incremental = false

# Execute benchmark tests with debug checks, without release-mode build costs.
[profile.ci-bench]
inherits = "ci"
opt-level = 1