Apache DataFusion SQL Query Engine
ci: add slt-only test coverage (#26219)
Yongting Youf8983afc96d1
diff --git a/.github/codecov.yml b/.github/codecov.ymlnew file mode 100644index 0000000000..926af5ef10--- /dev/null+++ b/.github/codecov.yml@@ -0,0 +1,54 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Configuration reference: https://docs.codecov.com/docs/codecovyml-reference++# Upload code coverage report, GitHub bot will comment with the code coverage report.+#+# It has two reports:+# - All-tests (default): code coverage from running all tests.+# - sqllogictests: code coverage from running sqllogictests.+coverage:+ status:+ # Coverage across the whole project.+ project:+ default:+ flags: [all-tests]+ sqllogictests:+ flags: [sqllogictests]+ informational: true+ # Coverage of changed lines.+ patch:+ default:+ flags: [all-tests]+ sqllogictests:+ flags: [sqllogictests]+ informational: true++# Show both reports in one expanded PR comment.+comment:+ layout: "flags, diff, files"+ behavior: default+ hide_comment_details: false+ hide_project_coverage: false++# Keep reports separate and use only coverage uploaded for this commit.+flags:+ all-tests:+ carryforward: false+ sqllogictests:+ carryforward: falsediff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex d07035b536..4db0dcacee 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -465,6 +465,10 @@ jobs:env:RUST_BACKTRACE: 1run: cargo xtask ci step test workspace+ - name: Run SQLLogicTests with coverage+ env:+ RUST_BACKTRACE: 1+ run: cargo xtask ci step test slt-coverage- parallel:- name: Verify Working Directory Cleanrun: git diff --exit-code@@ -477,10 +481,20 @@ jobs:echo "$STATUS"exit 1fi- - name: Upload coverage to codecov.io+ - name: Upload all-test coverage to codecov.iouses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1with:files: target/codecov.json+ flags: all-tests+ disable_search: true+ fail_ci_if_error: false+ token: ${{ secrets.CODECOV_TOKEN }}+ - name: Upload SQLLogicTest coverage to codecov.io+ uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1+ with:+ files: target/codecov-slt.json+ flags: sqllogictests+ disable_search: truefail_ci_if_error: falsetoken: ${{ secrets.CODECOV_TOKEN }}diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex 49a94a9689..fee30dfd4b 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -74,6 +74,26 @@ cargo nextest run[the book]: https://doc.rust-lang.org/book/[cargo-nextest]: https://nexte.st/+### Code coverage++The [Codecov] bot posts a coverage report on GitHub pull requests with:++- `all-tests`: coverage from the workspace test suite.+- `sqllogictests`: coverage from SQLLogicTests alone.++We [prefer end-to-end tests](#choosing-what-kind-of-test-to-write) because they are+easier to maintain and verify final results through the full query pipeline,+including optimizer passes. The SLT report helps ensure good coverage from these+tests alone.++Coverage helps find missing tests; high coverage does not guarantee good tests:++- **Do:** Enumerate edge cases to build comprehensive tests, then check the+ coverage report for gaps.+- **Don't:** Use Codecov to choose a minimal set of tests that covers a new feature.++[codecov]: https://app.codecov.io/gh/apache/datafusion+## Choosing What Kind of Test to WriteWhen adding tests, prefer tests that exercise **user-visible behavior** overdiff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex 1761df4ed0..5184a5cfbf 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -44,7 +44,7 @@ static CI_STEPS: &[StepInfo] = &[},StepInfo {command: "test",- help_usage: "test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",+ help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",help_examples: &["test cli", "test workspace"],help_description: "Run a CI test suite",error_message: "Cargo test step failed",@@ -218,7 +218,7 @@ impl StepContext {let mut command = self.cargo();match variant.as_str() {- "workspace" => {+ "workspace" | "slt-coverage" => {command.args(["llvm-cov","--profile",@@ -230,14 +230,20 @@ impl StepContext {"--exclude","datafusion-cli","--workspace",- "--lib",- "--tests",- "--bins",+ ]);+ let output_path = if variant == "slt-coverage" {+ command.args(["--test", "sqllogictests"]);+ "target/codecov-slt.json"+ } else {+ command.args(["--lib", "--tests", "--bins"]);+ "target/codecov.json"+ };+ command.args(["--features","serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait","--codecov","--output-path",- "target/codecov.json",+ output_path,]);}"cli" => {@@ -648,7 +654,7 @@ fn required_env(name: &str) -> Result<String> {mod tests {use super::*;- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";fn args(args: &[&str]) -> Vec<String> {args.iter().map(|arg| (*arg).to_string()).collect()@@ -836,6 +842,24 @@ cargo llvm-cov \--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \--codecov \--output-path target/codecov.json+")+ },+ },+ Cmd {+ command: &["test", "slt-coverage"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+cargo llvm-cov \+--profile ci \+--exclude datafusion-examples \+--exclude ffi_example_table_provider \+--exclude datafusion-cli \+--workspace \+--test sqllogictests \+--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \+--codecov \+--output-path target/codecov-slt.json")},},