Apache DataFusion SQL Query Engine
chore: Format examples in doc strings - crate datafusion (#18333)
Yu-Chuan Hung120c5bf843c1
diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rsindex 287a133273..3186c5cb82 100644--- a/datafusion/core/src/dataframe/mod.rs+++ b/datafusion/core/src/dataframe/mod.rs@@ -258,10 +258,13 @@ impl DataFrame {/// # async fn main() -> Result<()> {/// // datafusion will parse number as i64 first./// let sql = "a > 1 and b in (1, 10)";- /// let expected = col("a").gt(lit(1 as i64))- /// .and(col("b").in_list(vec![lit(1 as i64), lit(10 as i64)], false));+ /// let expected = col("a")+ /// .gt(lit(1 as i64))+ /// .and(col("b").in_list(vec![lit(1 as i64), lit(10 as i64)], false));/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let expr = df.parse_sql_expr(sql)?;/// assert_eq!(expected, expr);/// # Ok(())@@ -289,14 +292,16 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.select_columns(&["a", "b"])?;/// let expected = vec![/// "+---+---+",/// "| a | b |",/// "+---+---+",/// "| 1 | 2 |",- /// "+---+---+"+ /// "+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -329,8 +334,10 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let df : DataFrame = df.select_exprs(&["a * b", "c"])?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let df: DataFrame = df.select_exprs(&["a * b", "c"])?;/// # Ok(())/// # }/// ```@@ -357,14 +364,16 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.select(vec![col("a"), col("b") * col("c")])?;/// let expected = vec![/// "+---+-----------------------+",/// "| a | ?table?.b * ?table?.c |",/// "+---+-----------------------+",/// "| 1 | 6 |",- /// "+---+-----------------------+"+ /// "+---+-----------------------+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -407,7 +416,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// // +----+----+----+/// // | a | b | c |/// // +----+----+----+@@ -419,7 +430,7 @@ impl DataFrame {/// "| b | c |",/// "+---+---+",/// "| 2 | 3 |",- /// "+---+---+"+ /// "+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -518,7 +529,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.filter(col("a").lt_eq(col("b")))?;/// // all rows where a <= b are returned/// let expected = vec![@@ -528,7 +541,7 @@ impl DataFrame {/// "| 1 | 2 | 3 |",/// "| 4 | 5 | 6 |",/// "| 7 | 8 | 9 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -557,7 +570,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;////// // The following use is the equivalent of "SELECT MIN(b) GROUP BY a"/// let df1 = df.clone().aggregate(vec![col("a")], vec![min(col("b"))])?;@@ -568,7 +583,7 @@ impl DataFrame {/// "| 1 | 2 |",/// "| 4 | 5 |",/// "| 7 | 8 |",- /// "+---+----------------+"+ /// "+---+----------------+",/// ];/// assert_batches_sorted_eq!(expected1, &df1.collect().await?);/// // The following use is the equivalent of "SELECT MIN(b)"@@ -578,7 +593,7 @@ impl DataFrame {/// "| min(?table?.b) |",/// "+----------------+",/// "| 2 |",- /// "+----------------+"+ /// "+----------------+",/// ];/// # assert_batches_sorted_eq!(expected2, &df2.collect().await?);/// # Ok(())@@ -646,7 +661,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.limit(1, Some(2))?;/// let expected = vec![/// "+---+---+---+",@@ -654,7 +671,7 @@ impl DataFrame {/// "+---+---+---+",/// "| 4 | 5 | 6 |",/// "| 7 | 8 | 9 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -683,7 +700,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await? ;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let d2 = df.clone();/// let df = df.union(d2)?;/// let expected = vec![@@ -692,7 +711,7 @@ impl DataFrame {/// "+---+---+---+",/// "| 1 | 2 | 3 |",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -723,8 +742,13 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let d2 = df.clone().select_columns(&["b", "c", "a"])?.with_column("d", lit("77"))?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let d2 = df+ /// .clone()+ /// .select_columns(&["b", "c", "a"])?+ /// .with_column("d", lit("77"))?;/// let df = df.union_by_name(d2)?;/// let expected = vec![/// "+---+---+---+----+",@@ -732,7 +756,7 @@ impl DataFrame {/// "+---+---+---+----+",/// "| 1 | 2 | 3 | |",/// "| 1 | 2 | 3 | 77 |",- /// "+---+---+---+----+"+ /// "+---+---+---+----+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -762,7 +786,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let d2 = df.clone();/// let df = df.union_distinct(d2)?;/// // df2 are duplicate of df@@ -771,7 +797,7 @@ impl DataFrame {/// "| a | b | c |",/// "+---+---+---+",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -802,7 +828,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let d2 = df.clone().select_columns(&["b", "c", "a"])?;/// let df = df.union_by_name_distinct(d2)?;/// let expected = vec![@@ -810,7 +838,7 @@ impl DataFrame {/// "| a | b | c |",/// "+---+---+---+",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -837,14 +865,16 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.distinct()?;/// let expected = vec![/// "+---+---+---+",/// "| a | b | c |",/// "+---+---+---+",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -871,15 +901,17 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?- /// // Return a single row (a, b) for each distinct value of a- /// .distinct_on(vec![col("a")], vec![col("a"), col("b")], None)?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?+ /// // Return a single row (a, b) for each distinct value of a+ /// .distinct_on(vec![col("a")], vec![col("a"), col("b")], None)?;/// let expected = vec![/// "+---+---+",/// "| a | b |",/// "+---+---+",/// "| 1 | 2 |",- /// "+---+---+"+ /// "+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -1125,11 +1157,13 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.sort(vec![- /// col("a").sort(false, true), // a DESC, nulls first- /// col("b").sort(true, false), // b ASC, nulls last- /// ])?;+ /// col("a").sort(false, true), // a DESC, nulls first+ /// col("b").sort(true, false), // b ASC, nulls last+ /// ])?;/// let expected = vec![/// "+---+---+---+",/// "| a | b | c |",@@ -1176,12 +1210,17 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let left = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let right = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?- /// .select(vec![- /// col("a").alias("a2"),- /// col("b").alias("b2"),- /// col("c").alias("c2")])?;+ /// let left = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let right = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?+ /// .select(vec![+ /// col("a").alias("a2"),+ /// col("b").alias("b2"),+ /// col("c").alias("c2"),+ /// ])?;/// // Perform the equivalent of `left INNER JOIN right ON (a = a2 AND b = b2)`/// // finding all pairs of rows from `left` and `right` where `a = a2` and `b = b2`./// let join = left.join(right, JoinType::Inner, &["a", "b"], &["a2", "b2"], None)?;@@ -1190,13 +1229,12 @@ impl DataFrame {/// "| a | b | c | a2 | b2 | c2 |",/// "+---+---+---+----+----+----+",/// "| 1 | 2 | 3 | 1 | 2 | 3 |",- /// "+---+---+---+----+----+----+"+ /// "+---+---+---+----+----+----+",/// ];/// assert_batches_sorted_eq!(expected, &join.collect().await?);/// # Ok(())/// # }/// ```- ///pub fn join(self,right: DataFrame,@@ -1258,7 +1296,7 @@ impl DataFrame {/// "+---+---+---+----+----+----+",/// "| a | b | c | a2 | b2 | c2 |",/// "+---+---+---+----+----+----+",- /// "+---+---+---+----+----+----+"+ /// "+---+---+---+----+----+----+",/// ];/// # assert_batches_sorted_eq!(expected, &join_on.collect().await?);/// # Ok(())@@ -1290,7 +1328,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df1 = df.repartition(Partitioning::RoundRobinBatch(4))?;/// let expected = vec![/// "+---+---+---+",@@ -1299,7 +1339,7 @@ impl DataFrame {/// "| 1 | 2 | 3 |",/// "| 4 | 5 | 6 |",/// "| 7 | 8 | 9 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df1.collect().await?);/// # Ok(())@@ -1328,7 +1368,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let count = df.count().await?; // 1/// # assert_eq!(count, 1);/// # Ok(())@@ -1367,7 +1409,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let batches = df.collect().await?;/// # Ok(())/// # }@@ -1387,7 +1431,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// df.show().await?;/// # Ok(())/// # }@@ -1446,7 +1492,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// df.show_limit(10).await?;/// # Ok(())/// # }@@ -1472,7 +1520,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let stream = df.execute_stream().await?;/// # Ok(())/// # }@@ -1498,7 +1548,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let batches = df.collect_partitioned().await?;/// # Ok(())/// # }@@ -1518,7 +1570,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let batches = df.execute_stream_partitioned().await?;/// # Ok(())/// # }@@ -1547,7 +1601,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let schema = df.schema();/// # Ok(())/// # }@@ -1613,8 +1669,14 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let batches = df.limit(0, Some(100))?.explain(false, false)?.collect().await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let batches = df+ /// .limit(0, Some(100))?+ /// .explain(false, false)?+ /// .collect()+ /// .await?;/// # Ok(())/// # }/// ```@@ -1637,8 +1699,18 @@ impl DataFrame {/// # async fn main() -> Result<()> {/// use datafusion_expr::{Explain, ExplainOption};/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let batches = df.limit(0, Some(100))?.explain_with_options(ExplainOption::default().with_verbose(false).with_analyze(false))?.collect().await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let batches = df+ /// .limit(0, Some(100))?+ /// .explain_with_options(+ /// ExplainOption::default()+ /// .with_verbose(false)+ /// .with_analyze(false),+ /// )?+ /// .collect()+ /// .await?;/// # Ok(())/// # }/// ```@@ -1668,7 +1740,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let f = df.registry();/// // use f.udf("name", vec![...]) to use the udf/// # Ok(())@@ -1687,15 +1761,19 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let d2 = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let d2 = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.intersect(d2)?;/// let expected = vec![/// "+---+---+---+",/// "| a | b | c |",/// "+---+---+---+",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -1721,15 +1799,19 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;- /// let d2 = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;+ /// let d2 = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.intersect_distinct(d2)?;/// let expected = vec![/// "+---+---+---+",/// "| a | b | c |",/// "+---+---+---+",/// "| 1 | 2 | 3 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &df.collect().await?);/// # Ok(())@@ -1755,8 +1837,12 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;- /// let d2 = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;+ /// let d2 = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let result = df.except(d2)?;/// // those columns are not in example.csv, but in example_long.csv/// let expected = vec![@@ -1765,7 +1851,7 @@ impl DataFrame {/// "+---+---+---+",/// "| 4 | 5 | 6 |",/// "| 7 | 8 | 9 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &result.collect().await?);/// # Ok(())@@ -1791,8 +1877,12 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example_long.csv", CsvReadOptions::new()).await?;- /// let d2 = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example_long.csv", CsvReadOptions::new())+ /// .await?;+ /// let d2 = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let result = df.except_distinct(d2)?;/// // those columns are not in example.csv, but in example_long.csv/// let expected = vec![@@ -1801,7 +1891,7 @@ impl DataFrame {/// "+---+---+---+",/// "| 4 | 5 | 6 |",/// "| 7 | 8 | 9 |",- /// "+---+---+---+"+ /// "+---+---+---+",/// ];/// # assert_batches_sorted_eq!(expected, &result.collect().await?);/// # Ok(())@@ -1878,13 +1968,15 @@ impl DataFrame {/// use datafusion::dataframe::DataFrameWriteOptions;/// let ctx = SessionContext::new();/// // Sort the data by column "b" and write it to a new location- /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?- /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first- /// .write_csv(- /// "output.csv",- /// DataFrameWriteOptions::new(),- /// None, // can also specify CSV writing options here- /// ).await?;+ /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?+ /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first+ /// .write_csv(+ /// "output.csv",+ /// DataFrameWriteOptions::new(),+ /// None, // can also specify CSV writing options here+ /// )+ /// .await?;/// # fs::remove_file("output.csv")?;/// # Ok(())/// # }@@ -1948,13 +2040,11 @@ impl DataFrame {/// use datafusion::dataframe::DataFrameWriteOptions;/// let ctx = SessionContext::new();/// // Sort the data by column "b" and write it to a new location- /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?- /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first- /// .write_json(- /// "output.json",- /// DataFrameWriteOptions::new(),- /// None- /// ).await?;+ /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?+ /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first+ /// .write_json("output.json", DataFrameWriteOptions::new(), None)+ /// .await?;/// # fs::remove_file("output.json")?;/// # Ok(())/// # }@@ -2015,7 +2105,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.with_column("ab_sum", col("a") + col("b"))?;/// # Ok(())/// # }@@ -2089,7 +2181,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.with_column_renamed("ab_sum", "total")?;////// # Ok(())@@ -2222,7 +2316,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// let df = df.cache().await?;/// # Ok(())/// # }@@ -2266,7 +2362,9 @@ impl DataFrame {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// // Fill nulls in only columns "a" and "c":/// let df = df.fill_null(ScalarValue::from(0), vec!["a".to_owned(), "c".to_owned()])?;/// // Fill nulls across all columns:@@ -2337,9 +2435,9 @@ impl DataFrame {/// Helper for creating DataFrame./// # Example/// ```- /// use std::sync::Arc;/// use arrow::array::{ArrayRef, Int32Array, StringArray};/// use datafusion::prelude::DataFrame;+ /// use std::sync::Arc;/// let id: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));/// let name: ArrayRef = Arc::new(StringArray::from(vec!["foo", "bar", "baz"]));/// let df = DataFrame::from_columns(vec![("id", id), ("name", name)]).unwrap();diff --git a/datafusion/core/src/dataframe/parquet.rs b/datafusion/core/src/dataframe/parquet.rsindex 930b4fad1d..cb8a6cf295 100644--- a/datafusion/core/src/dataframe/parquet.rs+++ b/datafusion/core/src/dataframe/parquet.rs@@ -42,13 +42,15 @@ impl DataFrame {/// use datafusion::dataframe::DataFrameWriteOptions;/// let ctx = SessionContext::new();/// // Sort the data by column "b" and write it to a new location- /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?- /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first- /// .write_parquet(- /// "output.parquet",- /// DataFrameWriteOptions::new(),- /// None, // can also specify parquet writing options here- /// ).await?;+ /// ctx.read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?+ /// .sort(vec![col("b").sort(true, true)])? // sort by b asc, nulls first+ /// .write_parquet(+ /// "output.parquet",+ /// DataFrameWriteOptions::new(),+ /// None, // can also specify parquet writing options here+ /// )+ /// .await?;/// # fs::remove_file("output.parquet")?;/// # Ok(())/// # }diff --git a/datafusion/core/src/execution/context/csv.rs b/datafusion/core/src/execution/context/csv.rsindex 15d6d21f03..e6f95886e9 100644--- a/datafusion/core/src/execution/context/csv.rs+++ b/datafusion/core/src/execution/context/csv.rs@@ -37,9 +37,16 @@ impl SessionContext {/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();/// // You can read a single file using `read_csv`- /// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv("tests/data/example.csv", CsvReadOptions::new())+ /// .await?;/// // you can also read multiple files:- /// let df = ctx.read_csv(vec!["tests/data/example.csv", "tests/data/example.csv"], CsvReadOptions::new()).await?;+ /// let df = ctx+ /// .read_csv(+ /// vec!["tests/data/example.csv", "tests/data/example.csv"],+ /// CsvReadOptions::new(),+ /// )+ /// .await?;/// # Ok(())/// # }/// ```diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rsindex 448ee5264a..687779787a 100644--- a/datafusion/core/src/execution/context/mod.rs+++ b/datafusion/core/src/execution/context/mod.rs@@ -166,22 +166,23 @@ where/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();-/// let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;-/// let df = df.filter(col("a").lt_eq(col("b")))?-/// .aggregate(vec![col("a")], vec![min(col("b"))])?-/// .limit(0, Some(100))?;-/// let results = df-/// .collect()-/// .await?;+/// let df = ctx+/// .read_csv("tests/data/example.csv", CsvReadOptions::new())+/// .await?;+/// let df = df+/// .filter(col("a").lt_eq(col("b")))?+/// .aggregate(vec![col("a")], vec![min(col("b"))])?+/// .limit(0, Some(100))?;+/// let results = df.collect().await?;/// assert_batches_eq!(-/// &[-/// "+---+----------------+",-/// "| a | min(?table?.b) |",-/// "+---+----------------+",-/// "| 1 | 2 |",-/// "+---+----------------+",-/// ],-/// &results+/// &[+/// "+---+----------------+",+/// "| a | min(?table?.b) |",+/// "+---+----------------+",+/// "| 1 | 2 |",+/// "+---+----------------+",+/// ],+/// &results/// );/// # Ok(())/// # }@@ -197,21 +198,22 @@ where/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();-/// ctx.register_csv("example", "tests/data/example.csv", CsvReadOptions::new()).await?;+/// ctx.register_csv("example", "tests/data/example.csv", CsvReadOptions::new())+/// .await?;/// let results = ctx-/// .sql("SELECT a, min(b) FROM example GROUP BY a LIMIT 100")-/// .await?-/// .collect()-/// .await?;+/// .sql("SELECT a, min(b) FROM example GROUP BY a LIMIT 100")+/// .await?+/// .collect()+/// .await?;/// assert_batches_eq!(-/// &[-/// "+---+----------------+",-/// "| a | min(example.b) |",-/// "+---+----------------+",-/// "| 1 | 2 |",-/// "+---+----------------+",-/// ],-/// &results+/// &[+/// "+---+----------------+",+/// "| a | min(example.b) |",+/// "+---+----------------+",+/// "| 1 | 2 |",+/// "+---+----------------+",+/// ],+/// &results/// );/// # Ok(())/// # }@@ -231,18 +233,18 @@ where/// let config = SessionConfig::new().with_batch_size(4 * 1024);////// // configure a memory limit of 1GB with 20% slop-/// let runtime_env = RuntimeEnvBuilder::new()+/// let runtime_env = RuntimeEnvBuilder::new()/// .with_memory_limit(1024 * 1024 * 1024, 0.80)/// .build_arc()/// .unwrap();////// // Create a SessionState using the config and runtime_env/// let state = SessionStateBuilder::new()-/// .with_config(config)-/// .with_runtime_env(runtime_env)-/// // include support for built in functions and configurations-/// .with_default_features()-/// .build();+/// .with_config(config)+/// .with_runtime_env(runtime_env)+/// // include support for built in functions and configurations+/// .with_default_features()+/// .build();////// // Create a SessionContext/// let ctx = SessionContext::from(state);@@ -428,16 +430,14 @@ impl SessionContext {/// # use datafusion::prelude::*;/// # use datafusion::execution::SessionStateBuilder;/// # use datafusion_optimizer::push_down_filter::PushDownFilter;- /// let my_rule = PushDownFilter{}; // pretend it is a new rule- /// // Create a new builder with a custom optimizer rule+ /// let my_rule = PushDownFilter {}; // pretend it is a new rule+ /// // Create a new builder with a custom optimizer rule/// let context: SessionContext = SessionStateBuilder::new()- /// .with_optimizer_rule(Arc::new(my_rule))- /// .build()- /// .into();+ /// .with_optimizer_rule(Arc::new(my_rule))+ /// .build()+ /// .into();/// // Enable local file access and convert context back to a builder- /// let builder = context- /// .enable_url_table()- /// .into_state_builder();+ /// let builder = context.enable_url_table().into_state_builder();/// ```pub fn into_state_builder(self) -> SessionStateBuilder {let SessionContext {@@ -585,11 +585,10 @@ impl SessionContext {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// ctx- /// .sql("CREATE TABLE foo (x INTEGER)")- /// .await?- /// .collect()- /// .await?;+ /// ctx.sql("CREATE TABLE foo (x INTEGER)")+ /// .await?+ /// .collect()+ /// .await?;/// assert!(ctx.table_exist("foo").unwrap());/// # Ok(())/// # }@@ -614,14 +613,14 @@ impl SessionContext {/// # #[tokio::main]/// # async fn main() -> Result<()> {/// let ctx = SessionContext::new();- /// let options = SQLOptions::new()- /// .with_allow_ddl(false);- /// let err = ctx.sql_with_options("CREATE TABLE foo (x INTEGER)", options)- /// .await- /// .unwrap_err();- /// assert!(- /// err.to_string().starts_with("Error during planning: DDL not supported: CreateMemoryTable")- /// );+ /// let options = SQLOptions::new().with_allow_ddl(false);+ /// let err = ctx+ /// .sql_with_options("CREATE TABLE foo (x INTEGER)", options)+ /// .await+ /// .unwrap_err();+ /// assert!(err+ /// .to_string()+ /// .starts_with("Error during planning: DDL not supported: CreateMemoryTable"));/// # Ok(())/// # }/// ```@@ -653,8 +652,7 @@ impl SessionContext {/// // provide type information that `a` is an Int32/// let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);/// let df_schema = DFSchema::try_from(schema).unwrap();- /// let expr = SessionContext::new()- /// .parse_sql_expr(sql, &df_schema)?;+ /// let expr = SessionContext::new().parse_sql_expr(sql, &df_schema)?;/// assert_eq!(expected, expr);/// # Ok(())/// # }@@ -1143,8 +1141,14 @@ impl SessionContext {/// ```/// use datafusion::execution::context::SessionContext;///- /// assert_eq!(SessionContext::parse_memory_limit("1M").unwrap(), 1024 * 1024);- /// assert_eq!(SessionContext::parse_memory_limit("1.5G").unwrap(), (1.5 * 1024.0 * 1024.0 * 1024.0) as usize);+ /// assert_eq!(+ /// SessionContext::parse_memory_limit("1M").unwrap(),+ /// 1024 * 1024+ /// );+ /// assert_eq!(+ /// SessionContext::parse_memory_limit("1.5G").unwrap(),+ /// (1.5 * 1024.0 * 1024.0 * 1024.0) as usize+ /// );/// ```pub fn parse_memory_limit(limit: &str) -> Result<usize> {let (number, unit) = limit.split_at(limit.len() - 1);diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rsindex 561e0c363a..2949b17537 100644--- a/datafusion/core/src/execution/session_state.rs+++ b/datafusion/core/src/execution/session_state.rs@@ -114,12 +114,12 @@ use uuid::Uuid;/// # use std::sync::Arc;/// # #[tokio::main]/// # async fn main() -> Result<()> {-/// let state = SessionStateBuilder::new()-/// .with_config(SessionConfig::new())-/// .with_runtime_env(Arc::new(RuntimeEnv::default()))-/// .with_default_features()-/// .build();-/// Ok(())+/// let state = SessionStateBuilder::new()+/// .with_config(SessionConfig::new())+/// .with_runtime_env(Arc::new(RuntimeEnv::default()))+/// .with_default_features()+/// .build();+/// Ok(())/// # }/// ```///diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rsindex 78db28eaac..381dd5e9e8 100644--- a/datafusion/core/src/lib.rs+++ b/datafusion/core/src/lib.rs@@ -86,26 +86,29 @@//! let ctx = SessionContext::new();//!//! // create the dataframe-//! let df = ctx.read_csv("tests/data/example.csv", CsvReadOptions::new()).await?;+//! let df = ctx+//! .read_csv("tests/data/example.csv", CsvReadOptions::new())+//! .await?;//!//! // create a plan-//! let df = df.filter(col("a").lt_eq(col("b")))?-//! .aggregate(vec![col("a")], vec![min(col("b"))])?-//! .limit(0, Some(100))?;+//! let df = df+//! .filter(col("a").lt_eq(col("b")))?+//! .aggregate(vec![col("a")], vec![min(col("b"))])?+//! .limit(0, Some(100))?;//!//! // execute the plan//! let results: Vec<RecordBatch> = df.collect().await?;//!//! // format the results-//! let pretty_results = arrow::util::pretty::pretty_format_batches(&results)?-//! .to_string();+//! let pretty_results =+//! arrow::util::pretty::pretty_format_batches(&results)?.to_string();//!//! let expected = vec![//! "+---+----------------+",//! "| a | min(?table?.b) |",//! "+---+----------------+",//! "| 1 | 2 |",-//! "+---+----------------+"+//! "+---+----------------+",//! ];//!//! assert_eq!(pretty_results.trim().lines().collect::<Vec<_>>(), expected);@@ -126,24 +129,27 @@//! # async fn main() -> Result<()> {//! let ctx = SessionContext::new();//!-//! ctx.register_csv("example", "tests/data/example.csv", CsvReadOptions::new()).await?;+//! ctx.register_csv("example", "tests/data/example.csv", CsvReadOptions::new())+//! .await?;//!//! // create a plan-//! let df = ctx.sql("SELECT a, MIN(b) FROM example WHERE a <= b GROUP BY a LIMIT 100").await?;+//! let df = ctx+//! .sql("SELECT a, MIN(b) FROM example WHERE a <= b GROUP BY a LIMIT 100")+//! .await?;//!//! // execute the plan//! let results: Vec<RecordBatch> = df.collect().await?;//!//! // format the results-//! let pretty_results = arrow::util::pretty::pretty_format_batches(&results)?-//! .to_string();+//! let pretty_results =+//! arrow::util::pretty::pretty_format_batches(&results)?.to_string();//!//! let expected = vec![//! "+---+----------------+",//! "| a | min(example.b) |",//! "+---+----------------+",//! "| 1 | 2 |",-//! "+---+----------------+"+//! "+---+----------------+",//! ];//!//! assert_eq!(pretty_results.trim().lines().collect::<Vec<_>>(), expected);@@ -630,7 +636,7 @@//! └─────────────┘ ┗━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━┛//! ─────────────────────────────────────────────────────────────▶//! time-//!```+//! ```//!//! Note that DataFusion does not use [`tokio::task::spawn_blocking`] for//! CPU-bounded work, because `spawn_blocking` is designed for blocking **IO**,diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rsindex 2abfcd8417..fa8ea0b31c 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs@@ -44,7 +44,6 @@ use crate::fuzz_cases::aggregation_fuzzer::data_generator::Dataset;/// - hint `sorted` or not/// - `spilling` or not (TODO, I think a special `MemoryPool` may be needed/// to support this)-///pub struct SessionContextGenerator {/// Current testing datasetdataset: Arc<Dataset>,diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rsindex 753a74995d..aaf2d1b9ba 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs@@ -39,7 +39,6 @@ use crate::fuzz_cases::record_batch_generator::{ColumnDescr, RecordBatchGenerato/// will generate one `base dataset` firstly. Then the `base dataset` will be sorted/// based on each `sort_key` respectively. And finally `len(sort_keys) + 1` datasets/// will be returned-///#[derive(Debug, Clone)]pub struct DatasetGeneratorConfig {/// Descriptions of columns in datasets, it's `required`@@ -115,7 +114,6 @@ impl DatasetGeneratorConfig {////// - Split each batch to multiple batches which each sub-batch in has the randomly `rows num`,/// and this multiple batches will be used to create the `Dataset`.-///pub struct DatasetGenerator {batch_generator: RecordBatchGenerator,sort_keys_set: Vec<Vec<String>>,diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rsindex b90b3e5e32..1a8ef278cc 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs@@ -253,7 +253,6 @@ impl AggregationFuzzer {////// - `dataset_ref`, the input dataset, store it for error reported when found/// the inconsistency between the one for `ctx` and `expected results`.-///struct AggregationFuzzTestTask {/// Generated session context in current test casectx_with_params: SessionContextWithParams,diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rsindex 209278385b..766e2bedd7 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs@@ -24,7 +24,7 @@ use rand::{rng, seq::SliceRandom, Rng};/// Creates queries like/// ```sql/// SELECT AGG(..) FROM table_name GROUP BY <group_by_columns>-///```+/// ```#[derive(Debug, Default, Clone)]pub struct QueryBuilder {// ===================================@@ -95,7 +95,6 @@ pub struct QueryBuilder {/// More details can see [`GroupOrdering`].////// [`GroupOrdering`]: datafusion_physical_plan::aggregates::order::GroupOrdering- ///dataset_sort_keys: Vec<Vec<String>>,/// If we will also test the no grouping case like:@@ -103,7 +102,6 @@ pub struct QueryBuilder {/// ```text/// SELECT aggr FROM t;/// ```- ///no_grouping: bool,// ====================================diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rsindex e212ee269b..743c8750b5 100644--- a/datafusion/core/tests/sql/mod.rs+++ b/datafusion/core/tests/sql/mod.rs@@ -43,7 +43,6 @@ use tempfile::TempDir;/// A macro to assert that some particular line contains two substrings////// Usage: `assert_metrics!(actual, operator_name, metrics)`-///macro_rules! assert_metrics {($ACTUAL: expr, $OPERATOR_NAME: expr, $METRICS: expr) => {let found = $ACTUALdiff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex f0bf15d348..ffe0ba021e 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -56,7 +56,6 @@//!//! The same answer can be produced by simply keeping track of the top//! N elements, reducing the total amount of required buffer memory.-//!use std::fmt::Debug;use std::hash::Hash;