Skip to content

Apache DataFusion SQL Query Engine

ci: add slt-only test coverage (#26219)

Yongting Youf8983afc96d1
diff --git a/.github/codecov.yml b/.github/codecov.yml
new file mode 100644
index 0000000000..926af5ef10
--- /dev/null
+++ b/.github/codecov.yml
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Configuration reference: https://docs.codecov.com/docs/codecovyml-reference
+
+# Upload code coverage report, GitHub bot will comment with the code coverage report.
+#
+# It has two reports:
+# - All-tests (default): code coverage from running all tests.
+# - sqllogictests: code coverage from running sqllogictests.
+coverage:
+ status:
+ # Coverage across the whole project.
+ project:
+ default:
+ flags: [all-tests]
+ sqllogictests:
+ flags: [sqllogictests]
+ informational: true
+ # Coverage of changed lines.
+ patch:
+ default:
+ flags: [all-tests]
+ sqllogictests:
+ flags: [sqllogictests]
+ informational: true
+
+# Show both reports in one expanded PR comment.
+comment:
+ layout: "flags, diff, files"
+ behavior: default
+ hide_comment_details: false
+ hide_project_coverage: false
+
+# Keep reports separate and use only coverage uploaded for this commit.
+flags:
+ all-tests:
+ carryforward: false
+ sqllogictests:
+ carryforward: false
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index d07035b536..4db0dcacee 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -465,6 +465,10 @@ jobs:
env:
RUST_BACKTRACE: 1
run: cargo xtask ci step test workspace
+ - name: Run SQLLogicTests with coverage
+ env:
+ RUST_BACKTRACE: 1
+ run: cargo xtask ci step test slt-coverage
- parallel:
- name: Verify Working Directory Clean
run: git diff --exit-code
@@ -477,10 +481,20 @@ jobs:
echo "$STATUS"
exit 1
fi
- - name: Upload coverage to codecov.io
+ - name: Upload all-test coverage to codecov.io
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
files: target/codecov.json
+ flags: all-tests
+ disable_search: true
+ fail_ci_if_error: false
+ token: ${{ secrets.CODECOV_TOKEN }}
+ - name: Upload SQLLogicTest coverage to codecov.io
+ uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+ with:
+ files: target/codecov-slt.json
+ flags: sqllogictests
+ disable_search: true
fail_ci_if_error: false
token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index 49a94a9689..fee30dfd4b 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -74,6 +74,26 @@ cargo nextest run
[the book]: https://doc.rust-lang.org/book/
[cargo-nextest]: https://nexte.st/
+### Code coverage
+
+The [Codecov] bot posts a coverage report on GitHub pull requests with:
+
+- `all-tests`: coverage from the workspace test suite.
+- `sqllogictests`: coverage from SQLLogicTests alone.
+
+We [prefer end-to-end tests](#choosing-what-kind-of-test-to-write) because they are
+easier to maintain and verify final results through the full query pipeline,
+including optimizer passes. The SLT report helps ensure good coverage from these
+tests alone.
+
+Coverage helps find missing tests; high coverage does not guarantee good tests:
+
+- **Do:** Enumerate edge cases to build comprehensive tests, then check the
+ coverage report for gaps.
+- **Don't:** Use Codecov to choose a minimal set of tests that covers a new feature.
+
+[codecov]: https://app.codecov.io/gh/apache/datafusion
+
## Choosing What Kind of Test to Write
When adding tests, prefer tests that exercise **user-visible behavior** over
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
index 1761df4ed0..5184a5cfbf 100644
--- a/xtask/src/ci_steps.rs
+++ b/xtask/src/ci_steps.rs
@@ -44,7 +44,7 @@ static CI_STEPS: &[StepInfo] = &[
},
StepInfo {
command: "test",
- help_usage: "test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",
+ help_usage: "test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]",
help_examples: &["test cli", "test workspace"],
help_description: "Run a CI test suite",
error_message: "Cargo test step failed",
@@ -218,7 +218,7 @@ impl StepContext {
let mut command = self.cargo();
match variant.as_str() {
- "workspace" => {
+ "workspace" | "slt-coverage" => {
command.args([
"llvm-cov",
"--profile",
@@ -230,14 +230,20 @@ impl StepContext {
"--exclude",
"datafusion-cli",
"--workspace",
- "--lib",
- "--tests",
- "--bins",
+ ]);
+ let output_path = if variant == "slt-coverage" {
+ command.args(["--test", "sqllogictests"]);
+ "target/codecov-slt.json"
+ } else {
+ command.args(["--lib", "--tests", "--bins"]);
+ "target/codecov.json"
+ };
+ command.args([
"--features",
"serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait",
"--codecov",
"--output-path",
- "target/codecov.json",
+ output_path,
]);
}
"cli" => {
@@ -648,7 +654,7 @@ fn required_env(name: &str) -> Result<String> {
mod tests {
use super::*;
- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";
+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|slt-coverage|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|spill-pool-fuzz|hash-collisions|sqlite> [--explain]";
fn args(args: &[&str]) -> Vec<String> {
args.iter().map(|arg| (*arg).to_string()).collect()
@@ -836,6 +842,24 @@ cargo llvm-cov \
--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \
--codecov \
--output-path target/codecov.json
+")
+ },
+ },
+ Cmd {
+ command: &["test", "slt-coverage"],
+ expected: |actual| {
+ insta::assert_snapshot!(actual, @r"
+cd /workspace && \
+cargo llvm-cov \
+--profile ci \
+--exclude datafusion-examples \
+--exclude ffi_example_table_provider \
+--exclude datafusion-cli \
+--workspace \
+--test sqllogictests \
+--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait \
+--codecov \
+--output-path target/codecov-slt.json
")
},
},