Skip to content

Apache DataFusion SQL Query Engine

Add wide-schema benchmark suite for measuring per-file metadata overhead (#21970)

Adrian Garcia Badaracco1a934d1f5ef9
diff --git a/benchmarks/README.md b/benchmarks/README.md
index a4ddb09e07..d143de662e 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -620,6 +620,59 @@ This benchmarks is derived from the [TPC-H][1] version
[2]: https://github.com/databricks/tpch-dbgen.git,
[2.17.1]: https://www.tpc.org/tpc_documents_current_versions/pdf/tpc-h_v2.17.1.pdf
+## Wide-schema benchmark
+
+`wide_schema` measures the per-file metadata overhead of a wide schema
+in selective parquet scans — the regime where most of the work is
+loading footers / column-chunk metadata rather than reading row data,
+and that cost scales linearly with the number of column chunks in the
+dataset rather than with the number of columns the query references.
+
+The suite has two subgroups, selected via `BENCH_SUBGROUP`:
+
+- **`wide`** — runs against a 1024-col synthetic dataset. This is the
+ actual workload.
+- **`narrow`** — runs the same SQL against an 8-col version of the same
+ dataset (same row count, file count, per-file row-group shape).
+ This subgroup exists **only as a baseline for the wide subgroup** —
+ reading its numbers in isolation tells you very little. The
+ per-query wide-vs-narrow ratio is what isolates the schema-width
+ cost.
+
+All queries reference only base columns (no suffix-renamed copies),
+so each one runs on both subgroups and produces a directly comparable
+wide-vs-narrow pair.
+
+The data preparation step (`gen_wide_data`) synthesizes a generic
+8-column base schema (`id`, `value`, `count`, `ts`, `category`,
+`flag`, `status`, `text`) with deterministic data, then replicates it
+128× via suffix renaming (`_2`, `_3`, …) for 1024 columns total —
+written across 256 files at 50 k rows per file with one row group per
+file and ZSTD(1) compression. Copies 2..128 are zero-filled arrays so
+the schema is wide (every column still has its own footer / page
+index / column-chunk metadata) but the on-disk size stays around
+225 MB. The narrow dataset is written the same way without the suffix
+copies. The only variable between wide and narrow is schema width.
+
+```shell
+./benchmarks/bench.sh data wide_schema # synthesizes wide (1024 cols × 256 files) + narrow (8 cols × 256 files), ~60 s, ~335 MB
+./benchmarks/bench.sh run wide_schema # runs both 'wide' and 'narrow' subgroups; compare the per-query times for the slowdown ratio
+```
+
+The queries are deliberately small-projection (touch ≤ 4 columns) so
+the wide-schema overhead is the dominant signal. Coverage:
+
+- `Q01` — filter + project + `ORDER BY` + `LIMIT` (TopK shortcut)
+- `Q02` — project 1 column with a tight filter and `LIMIT 1`
+- `Q03` — tight filter + small projection, no sort
+- `Q04` — two low-cardinality string filters + a non-stat-prunable
+ modulo predicate for tight selectivity, project two columns, no
+ `LIMIT` or `ORDER BY`
+
+For cold-start measurements that include planner setup (the regime
+where this overhead is most visible), invoke `datafusion-cli`
+directly against `data/wide_schema/{wide,narrow}/`.
+
## TPCDS
Run the tpcds benchmark.
diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
index 4e7d743997..7a9a5fcdf3 100755
--- a/benchmarks/bench.sh
+++ b/benchmarks/bench.sh
@@ -100,6 +100,8 @@ sort_tpch: Benchmark of sorting speed for end-to-end sort queries o
sort_tpch10: Benchmark of sorting speed for end-to-end sort queries on TPC-H dataset (SF=10)
topk_tpch: Benchmark of top-k (sorting with limit) queries on TPC-H dataset (SF=1)
external_aggr: External aggregation benchmark on TPC-H dataset (SF=1)
+wide_schema: Small-projection queries on a wide synthetic dataset (1024 cols × 256 files) — measures per-file metadata overhead
+ (runs both 'wide' and 'narrow' subgroups: narrow is an internal baseline; the wide-vs-narrow ratio is the signal)
# ClickBench Benchmarks
clickbench_1: ClickBench queries against a single parquet file
@@ -239,6 +241,9 @@ main() {
tpch_csv10)
data_tpch "10" "csv"
;;
+ wide_schema)
+ data_wide_schema
+ ;;
tpcds)
data_tpcds
;;
@@ -444,6 +449,9 @@ main() {
tpch_mem10)
run_tpch_mem "10"
;;
+ wide_schema)
+ run_wide_schema
+ ;;
tpcds)
run_tpcds
;;
@@ -699,6 +707,68 @@ run_tpch() {
bash -c "$SQL_CARGO_COMMAND"
}
+# Synthesizes two parquet datasets used to measure per-file metadata
+# overhead of a wide schema:
+#
+# - data/wide_schema/wide/ 1024-col events × 256 files (~225 MB)
+# - data/wide_schema/narrow/ 8-col events × 256 files (~110 MB)
+#
+# Both share row count, file count, and per-file row-group shape; only
+# schema width differs. No external data source required — gen_wide_data
+# synthesizes everything from scratch in ~60 s.
+data_wide_schema() {
+ NUM_FILES=256
+ ROWS_PER_FILE=50000
+ WIDTH_FACTOR=128
+
+ DST_DIR="${DATA_DIR}/wide_schema"
+ WIDE_DIR="${DST_DIR}/wide"
+ NARROW_DIR="${DST_DIR}/narrow"
+
+ if [ -d "${WIDE_DIR}" ] && [ "$(ls -A "${WIDE_DIR}" 2>/dev/null | wc -l)" -ge ${NUM_FILES} ]; then
+ echo " wide parquet exists (${WIDE_DIR})."
+ else
+ mkdir -p "${WIDE_DIR}"
+ echo " synthesizing wide -> ${WIDE_DIR} (factor ${WIDTH_FACTOR}, ${NUM_FILES} files × ${ROWS_PER_FILE} rows) ..."
+ debug_run $CARGO_COMMAND --bin gen_wide_data -- \
+ --dst-dir "${WIDE_DIR}" \
+ --width-factor ${WIDTH_FACTOR} \
+ --num-files ${NUM_FILES} \
+ --rows-per-file ${ROWS_PER_FILE}
+ fi
+
+ if [ -d "${NARROW_DIR}" ] && [ "$(ls -A "${NARROW_DIR}" 2>/dev/null | wc -l)" -ge ${NUM_FILES} ]; then
+ echo " narrow parquet exists (${NARROW_DIR})."
+ else
+ mkdir -p "${NARROW_DIR}"
+ echo " synthesizing narrow -> ${NARROW_DIR} (8 base cols, ${NUM_FILES} files × ${ROWS_PER_FILE} rows) ..."
+ debug_run $CARGO_COMMAND --bin gen_wide_data -- \
+ --dst-dir "${NARROW_DIR}" \
+ --width-factor 1 \
+ --num-files ${NUM_FILES} \
+ --rows-per-file ${ROWS_PER_FILE}
+ fi
+}
+
+# Runs the wide_schema benchmark. Each query has a `subgroup`
+# directive that picks up BENCH_SUBGROUP, so we invoke the framework
+# twice — once with subgroup=wide (the actual workload) and once with
+# subgroup=narrow (the baseline). The wide-only queries (Q02/Q08/Q11/Q12)
+# hardcode `subgroup wide`, so they're skipped on the narrow pass.
+run_wide_schema() {
+ echo "Running wide_schema benchmark (wide subgroup)..."
+ debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=wide \
+ SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
+ ${QUERY:+BENCH_QUERY="${QUERY}"} \
+ bash -c "$SQL_CARGO_COMMAND"
+
+ echo "Running wide_schema benchmark (narrow baseline subgroup)..."
+ debug_run env BENCH_NAME=wide_schema BENCH_SUBGROUP=narrow \
+ SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
+ ${QUERY:+BENCH_QUERY="${QUERY}"} \
+ bash -c "$SQL_CARGO_COMMAND"
+}
+
# Runs the tpch in memory (needs tpch parquet data)
run_tpch_mem() {
SCALE_FACTOR=$1
diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.md
index 5bbf2ba81b..1705cf0d2f 100644
--- a/benchmarks/sql_benchmarks/README.md
+++ b/benchmarks/sql_benchmarks/README.md
@@ -41,6 +41,7 @@ in the community:
| `taxi` | NYC taxi dataset benchmark |
| `tpcds` | TPC‑DS queries |
| `tpch` | TPC‑H queries |
+| `wide_schema` | Small-projection queries on a wide (1024-col, 256-file) synthetic dataset; runs `wide` + `narrow` subgroups for comparison |
# Running Benchmarks
diff --git a/benchmarks/sql_benchmarks/wide_schema/benchmarks/q01.benchmark b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q01.benchmark
new file mode 100644
index 0000000000..5a9f854da2
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q01.benchmark
@@ -0,0 +1,25 @@
+-- Filter on three low-cardinality columns, project four columns,
+-- ORDER BY + LIMIT (TopK shortcut). Runs on both wide and narrow
+-- datasets via BENCH_SUBGROUP.
+
+name Q01
+group wide_schema
+subgroup ${BENCH_SUBGROUP:-wide}
+
+load sql_benchmarks/wide_schema/init/load.sql
+
+assert I
+SELECT COUNT(*) > 0 from events;
+----
+true
+
+run
+SELECT id, ts, value, text
+FROM events
+WHERE category = 'c0'
+ AND flag = 'f0'
+ AND status = 's0'
+ORDER BY ts DESC
+LIMIT 100;
+
+cleanup sql_benchmarks/wide_schema/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/wide_schema/benchmarks/q02.benchmark b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q02.benchmark
new file mode 100644
index 0000000000..dfe587b8e1
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q02.benchmark
@@ -0,0 +1,22 @@
+-- Project 1 column with a very tight filter. Stresses minimum-
+-- projection pushdown over a wide schema. Runs on both wide and
+-- narrow datasets via BENCH_SUBGROUP.
+
+name Q02
+group wide_schema
+subgroup ${BENCH_SUBGROUP:-wide}
+
+load sql_benchmarks/wide_schema/init/load.sql
+
+assert I
+SELECT COUNT(*) > 0 from events;
+----
+true
+
+run
+SELECT value
+FROM events
+WHERE id = 12345
+LIMIT 1;
+
+cleanup sql_benchmarks/wide_schema/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/wide_schema/benchmarks/q03.benchmark b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q03.benchmark
new file mode 100644
index 0000000000..622e47e18e
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q03.benchmark
@@ -0,0 +1,23 @@
+-- Tight filter + small projection without ORDER BY / LIMIT — measures
+-- pure filter+project throughput against a wide schema (no TopK
+-- shortcut). The filter is tight so the result set stays small.
+-- Runs on both wide and narrow datasets via BENCH_SUBGROUP.
+
+name Q03
+group wide_schema
+subgroup ${BENCH_SUBGROUP:-wide}
+
+load sql_benchmarks/wide_schema/init/load.sql
+
+assert I
+SELECT COUNT(*) > 0 from events;
+----
+true
+
+run
+SELECT id, ts, value, text
+FROM events
+WHERE id = 12345
+ AND category = 'c0';
+
+cleanup sql_benchmarks/wide_schema/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/wide_schema/benchmarks/q04.benchmark b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q04.benchmark
new file mode 100644
index 0000000000..eb904b4e08
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/benchmarks/q04.benchmark
@@ -0,0 +1,24 @@
+-- Two low-cardinality string filters + a non-stat-prunable modulo
+-- predicate for tight selectivity (~0.005 % match rate), project two
+-- columns, no LIMIT, no ORDER BY. Runs on both wide and narrow
+-- datasets via BENCH_SUBGROUP.
+
+name Q04
+group wide_schema
+subgroup ${BENCH_SUBGROUP:-wide}
+
+load sql_benchmarks/wide_schema/init/load.sql
+
+assert I
+SELECT COUNT(*) > 0 from events;
+----
+true
+
+run
+SELECT id, ts
+FROM events
+WHERE category = 'c0'
+ AND flag = 'f0'
+ AND id % 1000 = 0;
+
+cleanup sql_benchmarks/wide_schema/init/cleanup.sql
diff --git a/benchmarks/sql_benchmarks/wide_schema/init/cleanup.sql b/benchmarks/sql_benchmarks/wide_schema/init/cleanup.sql
new file mode 100644
index 0000000000..653c740037
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/init/cleanup.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS events;
diff --git a/benchmarks/sql_benchmarks/wide_schema/init/load.sql b/benchmarks/sql_benchmarks/wide_schema/init/load.sql
new file mode 100644
index 0000000000..4fbcda1d58
--- /dev/null
+++ b/benchmarks/sql_benchmarks/wide_schema/init/load.sql
@@ -0,0 +1,6 @@
+-- Registers the events table, picking the dataset based on BENCH_SUBGROUP:
+--
+-- BENCH_SUBGROUP=wide → 1024-col synthetic dataset (the actual benchmark)
+-- BENCH_SUBGROUP=narrow → 8-col baseline (companion only — meaningful
+-- only when compared to the wide numbers)
+CREATE EXTERNAL TABLE events STORED AS PARQUET LOCATION 'data/wide_schema/${BENCH_SUBGROUP:-wide}/';
diff --git a/benchmarks/src/bin/gen_wide_data.rs b/benchmarks/src/bin/gen_wide_data.rs
new file mode 100644
index 0000000000..90be7378a5
--- /dev/null
+++ b/benchmarks/src/bin/gen_wide_data.rs
@@ -0,0 +1,351 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Synthesizes parquet datasets for the `wide_schema` / `narrow_schema`
+//! benchmark suites.
+//!
+//! The base schema has 8 columns with deterministic synthetic data:
+//!
+//! id Int64 monotonic primary key
+//! value Float64 numeric value (~5 figures)
+//! count Int64 auxiliary integer
+//! ts Date32 spread across ~6 years
+//! category Utf8 low-cardinality (7 values)
+//! flag Utf8 low-cardinality (3 values)
+//! status Utf8 low-cardinality (2 values)
+//! text Utf8 ~40-char synthetic string
+//!
+//! `--width-factor N` replicates the schema with `_2`, `_3`, …, `_N`
+//! suffix copies. The suffix-renamed copies are zero-filled arrays of
+//! the same datatype, laid out **before** the base columns in the
+//! output schema — so the base data columns sit at the end. This
+//! makes column lookup for the filter / project columns traverse
+//! past all the padding, exercising any per-column-position cost in
+//! the scanner / planner.
+//!
+//! Zero-filled rather than null because the parquet reader can
+//! shortcut on all-null statistics, muting the wide-schema slowdown
+//! by ~35 %. Despite the wide schema, every column still has its own
+//! footer / page index / column-chunk metadata; the on-disk size
+//! stays small — the point being to measure per-file metadata
+//! overhead, not row IO.
+//! Zero-filled was chosen over random data or other options because
+//! it’s simple to implement, avoids any questions about what
+//! the data distribution or shape should look like, and still achieves
+//! the goal of having a wide table.
+
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use arrow::array::{
+ ArrayRef, Date32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray,
+};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use clap::Parser;
+use datafusion::error::Result;
+use datafusion_common::exec_datafusion_err;
+use parquet::arrow::ArrowWriter;
+use parquet::basic::{Compression, ZstdLevel};
+use parquet::file::properties::WriterProperties;
+
+#[derive(Debug, Parser)]
+#[command(
+ name = "gen_wide_data",
+ about = "Synthesize a wide-schema parquet dataset for the wide_schema benchmark suite"
+)]
+struct Cli {
+ /// Destination directory. Output files are written as
+ /// `events_0001.parquet`, `events_0002.parquet`, …
+ #[arg(long)]
+ dst_dir: PathBuf,
+
+ /// Number of times to replicate each base column. Copy 1 keeps the
+ /// original column names; copies 2..N append `_2`, `_3`, …, all
+ /// stored as zero-filled arrays. With the 8-column base schema,
+ /// factor 128 yields 1024 columns. Set to 1 for the narrow
+ /// variant (no replication, base columns only).
+ #[arg(long, default_value_t = 128)]
+ width_factor: usize,
+
+ /// Number of output files to write. Each file gets one row group.
+ #[arg(long, default_value_t = 256)]
+ num_files: usize,
+
+ /// Rows per output file.
+ #[arg(long, default_value_t = 50_000)]
+ rows_per_file: usize,
+
+ /// Per-batch row count for synthesis. Smaller batches reduce peak
+ /// memory; doesn't change output.
+ #[arg(long, default_value_t = 16_384)]
+ batch_size: usize,
+}
+
+const CATEGORIES: &[&str] = &["c0", "c1", "c2", "c3", "c4", "c5", "c6"];
+const FLAGS: &[&str] = &["f0", "f1", "f2"];
+const STATUSES: &[&str] = &["s0", "s1"];
+
+/// Build a zero-filled array of the given datatype. Used for the
+/// suffix-renamed padding columns. Zero-filled rather than all-null so
+/// the parquet reader can't shortcut on null-array statistics — the
+/// wide-schema slowdown reproduces ~35 % wider with zeros than with
+/// nulls.
+fn zero_array(dt: &DataType, n: usize) -> ArrayRef {
+ match dt {
+ DataType::Int32 => {
+ Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(0i32, n)))
+ }
+ DataType::Int64 => {
+ Arc::new(Int64Array::from_iter_values(std::iter::repeat_n(0i64, n)))
+ }
+ DataType::Float64 => Arc::new(Float64Array::from_iter_values(
+ std::iter::repeat_n(0.0f64, n),
+ )),
+ DataType::Date32 => {
+ Arc::new(Date32Array::from_iter_values(std::iter::repeat_n(0i32, n)))
+ }
+ DataType::Utf8 => {
+ Arc::new(StringArray::from_iter_values(std::iter::repeat_n("", n)))
+ }
+ _ => panic!("zero_array: unsupported datatype {dt:?}"),
+ }
+}
+
+/// Eight-column base schema. All fields nullable so the schema is
+/// uniform across base and zero-filled replicated copies.
+fn base_schema() -> SchemaRef {
+ Arc::new(Schema::new(vec![
+ Field::new("id", DataType::Int64, true),
+ Field::new("value", DataType::Float64, true),
+ Field::new("count", DataType::Int64, true),
+ Field::new("ts", DataType::Date32, true),
+ Field::new("category", DataType::Utf8, true),
+ Field::new("flag", DataType::Utf8, true),
+ Field::new("status", DataType::Utf8, true),
+ Field::new("text", DataType::Utf8, true),
+ ]))
+}
+
+/// Synthesize one batch of length `n` covering rows `[start, start+n)`.
+fn synthesize_batch(start: usize, n: usize, schema: &SchemaRef) -> Result<RecordBatch> {
+ let id = Int64Array::from_iter_values((start..start + n).map(|i| i as i64));
+ let value = Float64Array::from_iter_values(
+ (start..start + n).map(|i| 900.0 + ((i as f64 * 13.7) % 99100.0)),
+ );
+ let count =
+ Int64Array::from_iter_values((start..start + n).map(|i| (i % 1000) as i64));
+ // Date32 spread across ~6 years (epoch days; 8035 ≈ 1992-01-01).
+ let ts = Date32Array::from_iter_values(
+ (start..start + n).map(|i| 8035 + ((i % 2200) as i32)),
+ );
+ let category = StringArray::from_iter_values(
+ (start..start + n).map(|i| CATEGORIES[i % CATEGORIES.len()]),
+ );
+ let flag =
+ StringArray::from_iter_values((start..start + n).map(|i| FLAGS[i % FLAGS.len()]));
+ let status = StringArray::from_iter_values(
+ (start..start + n).map(|i| STATUSES[i % STATUSES.len()]),
+ );
+ let text = StringArray::from_iter_values(
+ (start..start + n).map(|i| format!("synthetic event row {i:010} payload text")),
+ );
+
+ let cols: Vec<ArrayRef> = vec![
+ Arc::new(id),
+ Arc::new(value),
+ Arc::new(count),
+ Arc::new(ts),
+ Arc::new(category),
+ Arc::new(flag),
+ Arc::new(status),
+ Arc::new(text),
+ ];
+
+ RecordBatch::try_new(Arc::clone(schema), cols)
+ .map_err(|e| exec_datafusion_err!("building synthetic batch: {e}"))
+}
+
+/// Builds the wide schema by laying out the suffix-renamed zero-padded
+/// copies first and the unsuffixed base columns last. Putting the base
+/// columns at the *end* of the schema is deliberate — column lookup
+/// for the filter / project columns has to traverse past all the
+/// padding entries, exercising any per-column-position cost in the
+/// scanner / planner.
+fn widen_schema(src: &SchemaRef, factor: usize) -> SchemaRef {
+ let src_fields = src.fields();
+ let mut fields: Vec<Arc<Field>> = Vec::with_capacity(src_fields.len() * factor);
+ for copy in 2..=factor {
+ for f in src_fields {
+ fields.push(Arc::new(Field::new(
+ format!("{}_{}", f.name(), copy),
+ f.data_type().clone(),
+ true,
+ )));
+ }
+ }
+ for f in src_fields {
+ fields.push(Arc::new(Field::new(
+ f.name().clone(),
+ f.data_type().clone(),
+ f.is_nullable(),
+ )));
+ }
+ Arc::new(Schema::new(fields))
+}
+
+fn widen_batch(
+ batch: &RecordBatch,
+ wide_schema: &SchemaRef,
+ factor: usize,
+) -> Result<RecordBatch> {
+ let cols = batch.columns();
+ let n_rows = batch.num_rows();
+ let mut wide = Vec::with_capacity(cols.len() * factor);
+ // Zero-padded copies first…
+ for _ in 2..=factor {
+ for c in cols {
+ wide.push(zero_array(c.data_type(), n_rows));
+ }
+ }
+ // …then the base data columns at the end.
+ for c in cols {
+ wide.push(Arc::clone(c));
+ }
+ RecordBatch::try_new(Arc::clone(wide_schema), wide)
+ .map_err(|e| exec_datafusion_err!("building wide batch: {e}"))
+}
+
+fn open_writer(
+ path: &Path,
+ schema: SchemaRef,
+ rows_per_group: usize,
+) -> Result<ArrowWriter<File>> {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)
+ .map_err(|e| exec_datafusion_err!("creating {}: {e}", parent.display()))?;
+ }
+ let file = File::create(path)
+ .map_err(|e| exec_datafusion_err!("creating {}: {e}", path.display()))?;
+ let writer_props = WriterProperties::builder()
+ .set_max_row_group_row_count(Some(rows_per_group.max(1)))
+ .set_compression(Compression::ZSTD(ZstdLevel::try_new(1).unwrap()))
+ .build();
+ ArrowWriter::try_new(file, schema, Some(writer_props))
+ .map_err(|e| exec_datafusion_err!("creating ArrowWriter: {e}"))
+}
+
+fn main() -> Result<()> {
+ let cli = Cli::parse();
+ if cli.width_factor == 0 {
+ return Err(exec_datafusion_err!("--width-factor must be >= 1"));
+ }
+ if cli.num_files == 0 {
+ return Err(exec_datafusion_err!("--num-files must be >= 1"));
+ }
+ if cli.rows_per_file == 0 {
+ return Err(exec_datafusion_err!("--rows-per-file must be >= 1"));
+ }
+ if cli.batch_size == 0 {
+ return Err(exec_datafusion_err!("--batch-size must be >= 1"));
+ }
+
+ let raw_schema = base_schema();
+ let wide_schema = widen_schema(&raw_schema, cli.width_factor);
+ let cap_rows = cli.rows_per_file.saturating_mul(cli.num_files);
+
+ println!(
+ "Synthesizing: {} base cols × {} factor = {} cols × {} files of {} rows each (total {} rows, ZSTD(1), 1 row group/file).",
+ raw_schema.fields().len(),
+ cli.width_factor,
+ wide_schema.fields().len(),
+ cli.num_files,
+ cli.rows_per_file,
+ cap_rows,
+ );
+
+ std::fs::create_dir_all(&cli.dst_dir)
+ .map_err(|e| exec_datafusion_err!("creating {}: {e}", cli.dst_dir.display()))?;
+
+ let mut file_idx: usize = 0;
+ let mut writer: Option<ArrowWriter<File>> = None;
+ let mut rows_in_current: usize = 0;
+ let mut total_written: usize = 0;
+ let mut row_cursor: usize = 0;
+
+ 'outer: while total_written < cap_rows {
+ let batch_n = cli.batch_size.min(cap_rows - total_written);
+ let base_batch = synthesize_batch(row_cursor, batch_n, &raw_schema)?;
+ let wide_batch = widen_batch(&base_batch, &wide_schema, cli.width_factor)?;
+ row_cursor += batch_n;
+
+ let mut remaining = wide_batch;
+ while remaining.num_rows() > 0 {
+ if writer.is_none() {
+ if file_idx >= cli.num_files {
+ break 'outer;
+ }
+ file_idx += 1;
+ let path = if cli.num_files == 1 {
+ cli.dst_dir.join("events.parquet")
+ } else {
+ cli.dst_dir.join(format!("events_{file_idx:04}.parquet"))
+ };
+ writer = Some(open_writer(
+ &path,
+ Arc::clone(&wide_schema),
+ cli.rows_per_file,
+ )?);
+ rows_in_current = 0;
+ }
+ let space = cli.rows_per_file.saturating_sub(rows_in_current);
+ let take = remaining.num_rows().min(space.max(1));
+ let chunk = remaining.slice(0, take);
+ writer
+ .as_mut()
+ .unwrap()
+ .write(&chunk)
+ .map_err(|e| exec_datafusion_err!("writing chunk: {e}"))?;
+ rows_in_current += take;
+ total_written += take;
+ if take == remaining.num_rows() {
+ remaining = remaining.slice(0, 0);
+ } else {
+ remaining = remaining.slice(take, remaining.num_rows() - take);
+ }
+ if rows_in_current >= cli.rows_per_file {
+ writer
+ .take()
+ .unwrap()
+ .close()
+ .map_err(|e| exec_datafusion_err!("closing writer: {e}"))?;
+ }
+ }
+ }
+ if let Some(w) = writer.take() {
+ w.close()
+ .map_err(|e| exec_datafusion_err!("closing writer: {e}"))?;
+ }
+
+ println!(
+ "Wrote {} rows across {} files in {}",
+ total_written,
+ file_idx,
+ cli.dst_dir.display(),
+ );
+ Ok(())
+}