Apache Solr open-source search software
Speed up BATS tests (#4478)
Jan Høydahleaee0a20cbb7
diff --git a/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java b/solr/core/src/test/org/apache/solr/cli/PackageToolTest.javaindex e8cb48874b6..9282af37e52 100644--- a/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java+++ b/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java@@ -373,6 +373,63 @@ public class PackageToolTest extends SolrCloudTestCase {success);}+ /** Validates that collection existence is checked before package resolution. */+ @Test+ public void testDeployValidationMessages() throws Exception {+ String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();++ withBasicAuth(CollectionAdminRequest.createCollection("validation-test", "conf1", 1, 1))+ .processAndWait(cluster.getSolrClient(), 10);++ CLITestHelper.TestingRuntime captureRuntime = new CLITestHelper.TestingRuntime(true);+ PackageTool tool = new PackageTool(captureRuntime);++ // Collection exists but package does not — collection validation should pass,+ // package lookup should fail.+ tool.runTool(+ SolrCLI.processCommandLineArgs(+ tool,+ new String[] {+ "--solr-url",+ solrUrl,+ "deploy",+ "NONEXISTENT_PKG",+ "--collections",+ "validation-test",+ "--credentials",+ SecurityJson.USER_PASS+ }));+ String deployOut = captureRuntime.getOutput();+ assertFalse(+ "Should not complain about invalid collection", deployOut.contains("Invalid collection"));+ assertTrue(+ "Should report missing package",+ deployOut.contains("Package instance doesn't exist: NONEXISTENT_PKG:null"));++ captureRuntime.clearOutput();++ // Undeploy of a package that was never deployed should give a clear message.+ tool.runTool(+ SolrCLI.processCommandLineArgs(+ tool,+ new String[] {+ "--solr-url",+ solrUrl,+ "undeploy",+ "NONEXISTENT_PKG",+ "--collections",+ "validation-test",+ "--credentials",+ SecurityJson.USER_PASS+ }));+ String undeployOut = captureRuntime.getOutput();+ assertFalse(+ "Should not complain about invalid collection", undeployOut.contains("Invalid collection"));+ assertTrue(+ "Should report package not deployed",+ undeployOut.contains("Package NONEXISTENT_PKG not deployed on collection validation-test"));+ }+private void run(PackageTool tool, String[] args) throws Exception {int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args));assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res);diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradleindex 7320f2ebe2b..953adafe01d 100644--- a/solr/packaging/build.gradle+++ b/solr/packaging/build.gradle@@ -281,6 +281,10 @@ task integrationTests(type: BatsTask) {}}+ if (!Boolean.parseBoolean((String) project.findProperty('solr.bats.nightly'))) {+ filterTags = '!nightly'+ }+inputs.dir(distDir)inputs.dir('test') // Track .bats test files and helper as inputsoutputs.dir(integrationTestOutput)@@ -320,6 +324,10 @@ class BatsTask extends Exec {@Inputvar testFiles = []+ @Input+ @Optional+ String filterTags = null+@Option(option = "tests", description = "Sets test cases to be included")public void setTestNamePatterns(List<String> tests) {// TODO: bats --filter <regex>@@ -335,6 +343,9 @@ class BatsTask extends Exec {if (logger.isInfoEnabled()) {batsArgs << '--verbose-run'}+ if (filterTags) {+ batsArgs += ['--filter-tags', filterTags]+ }batsArgs += ['-T', '--print-output-on-failure', '--report-formatter', 'junit', '--output', "$project.buildDir/test-output"]// Note: tests to run must be listed after all other argumentsbatsArgs += testFiles.empty ? [testDir] : testFilesdiff --git a/solr/packaging/test/README.md b/solr/packaging/test/README.mdindex bd73d494589..e575502090b 100644--- a/solr/packaging/test/README.md+++ b/solr/packaging/test/README.md@@ -55,6 +55,24 @@ Some tests will start clusters or create collections,It is recommended that you install and run `shellcheck` to verify your test scripts and catch common mistakes before committing your changes.+## Nightly Tests++Some BATS tests are slow or require network access (e.g. downloading models) and are+excluded from regular runs by default. These tests use BATS native tagging via+`# bats test_tags=nightly` and are filtered out with `--filter-tags !nightly`.++To include nightly tests, pass the Gradle property `solr.bats.nightly`:++ ./gradlew integrationTests -Psolr.bats.nightly=true++When writing a new test that should only run in nightly builds, add the BATS tag comment+directly before the `@test` declaration:++ # bats test_tags=nightly+ @test "my slow test" {+ ...+ }+## Limitations1. Currently this test suite is only available for \*nix environments. Although, newerdiff --git a/solr/packaging/test/test_example.bats b/solr/packaging/test/test_example.batsindex a15ba4b6f62..a5b4108e549 100644--- a/solr/packaging/test/test_example.bats+++ b/solr/packaging/test/test_example.bats@@ -17,25 +17,36 @@load bats_helper-setup() {+setup_file() {common_clean_setup+ solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"+ solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000+ solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000+}++teardown_file() {+ common_setup+ solr stop --all >/dev/null 2>&1+}++setup() {+ common_setup}teardown() {# save a snapshot of SOLR_HOME for failed testssave_home_on_failure+}- solr stop --all >/dev/null 2>&1+@test "start -e cloud works with --no-prompt" {+ solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000+ solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000}@test "start -e cloud works with --jvm-opts" {- solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld"- solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000- solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000-run curl "http://localhost:${SOLR_PORT}/solr/admin/info/properties"assert_output --partial 'helloworld'-+run curl "http://localhost:${SOLR2_PORT}/solr/admin/info/properties"assert_output --partial 'helloworld'}diff --git a/solr/packaging/test/test_example_noprompt.bats b/solr/packaging/test/test_example_noprompt.batsdeleted file mode 100644index 19329a8041a..00000000000--- a/solr/packaging/test/test_example_noprompt.bats+++ /dev/null@@ -1,35 +0,0 @@-#!/usr/bin/env bats--# Licensed to the Apache Software Foundation (ASF) under one or more-# contributor license agreements. See the NOTICE file distributed with-# this work for additional information regarding copyright ownership.-# The ASF licenses this file to You under the Apache License, Version 2.0-# (the "License"); you may not use this file except in compliance with-# the License. You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--load bats_helper--setup() {- common_clean_setup-}--teardown() {- # save a snapshot of SOLR_HOME for failed tests- save_home_on_failure-- solr stop --all >/dev/null 2>&1-}--@test "start -e cloud works with --no-prompt" {- solr start -e cloud --no-prompt- solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000- solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000-}diff --git a/solr/packaging/test/test_healthcheck.bats b/solr/packaging/test/test_healthcheck.batsindex e8f6069cb08..5492fb8b514 100644--- a/solr/packaging/test/test_healthcheck.bats+++ b/solr/packaging/test/test_healthcheck.bats@@ -29,22 +29,18 @@ teardown() {}@test "healthcheck on cloud solr" {- solr start -e films- run solr healthcheck -c films+ solr start+ solr create -c healthcheck_test -d _default+ # Local+ run solr healthcheck -c healthcheck_testrefute_output --partial 'error'--}--@test "healthcheck on remote cloud solr" {- solr start -e films- run solr healthcheck -c films --solr-connection http://localhost:${SOLR_PORT}/solr+ # Remote+ run solr healthcheck -c healthcheck_test --solr-connection http://localhost:${SOLR_PORT}/solrrefute_output --partial 'error'-}@test "healthcheck errors on standalone solr" {- solr start --user-managed -e films- run solr healthcheck -c films+ solr start --user-managed+ run solr healthcheck -c healthcheck_testassert_output --partial 'Healthcheck tool only works in Solr Cloud mode'-}diff --git a/solr/packaging/test/test_opennlp.bats b/solr/packaging/test/test_opennlp.batsindex 69ee71253cf..4f71259682f 100644--- a/solr/packaging/test/test_opennlp.bats+++ b/solr/packaging/test/test_opennlp.bats@@ -42,8 +42,8 @@ teardown() {# long test as an "integration" test is something we decide is okay?# I also have dreams of incorporating this as code snippets in a Tutorial via the ascii doc tags# like we use for the SolrJ code snippets. That way we know the snippets continue to work!+# bats test_tags=nightly@test "Check lifecycle of sentiment classification" {-echo "Downloading onnx model and vocab..."mkdir -p ${SOLR_TIP}/models/sentiment/diff --git a/solr/packaging/test/test_packages.bats b/solr/packaging/test/test_packages.batsdeleted file mode 100644index 1f60523e53f..00000000000--- a/solr/packaging/test/test_packages.bats+++ /dev/null@@ -1,81 +0,0 @@-#!/usr/bin/env bats--# Licensed to the Apache Software Foundation (ASF) under one or more-# contributor license agreements. See the NOTICE file distributed with-# this work for additional information regarding copyright ownership.-# The ASF licenses this file to You under the Apache License, Version 2.0-# (the "License"); you may not use this file except in compliance with-# the License. You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--load bats_helper--setup() {- common_clean_setup-}--teardown() {- # save a snapshot of SOLR_HOME for failed tests- save_home_on_failure-- solr stop --all >/dev/null 2>&1-}--@test "lifecycle of package" {- run solr start -Dsolr.packages.enabled=true-- run solr package --help- assert_output --partial "Add a repository to Solr"-- run solr package list-available- assert_output --partial "Available packages:"-}--@test "deploying and undeploying a collection level package" {- run solr start -Dsolr.packages.enabled=true-- solr create -c foo-1.2-- # Deploy package - the package doesn't need to exist before the collection validation kicks in- run solr package deploy PACKAGE_NAME --collections foo-1.2- # assert_output --partial "Deployment successful"- refute_output --partial "Invalid collection"-- # Until PACKAGE_NAME refers to an actual installable package, this is as far as we get.- assert_output --partial "Package instance doesn't exist: PACKAGE_NAME:null"-- # Undeploy package- run solr package undeploy PACKAGE_NAME --collections foo-1.2- refute_output --partial "Invalid collection"- assert_output --partial "Package PACKAGE_NAME not deployed on collection foo-1.2"-}--# This test is useful if you are debugging/working with packages.-# We have disabled it for now since it depends on a live internet-# connection to run. This could be updated with a local Repo server if we had-# a package that is part of the Solr project to use.-@test "deploying and undeploying a cluster level package" {- skip "For developing package infra; requires a connection to github"-- run solr start -Dsolr.packages.enabled=true-- run solr package add-repo splainer "https://raw.githubusercontent.com/o19s/splainer/refs/heads/main/solr-splainer-package/repo/"- assert_output --partial "Added repository: splainer"-- run solr package list-available- assert_output --partial "solr-splainer Splainer for Solr"- run solr package install solr-splainer- assert_output --partial "solr-splainer installed."-- run solr package deploy solr-splainer -y --cluster- assert_output --partial "Deployment successful"-- run -0 curl --fail http://localhost:${SOLR_PORT}/v2/splainer/index.html-}diff --git a/solr/packaging/test/test_ssl.bats b/solr/packaging/test/test_ssl.batsindex 5075ec76338..cf92d144d4f 100644--- a/solr/packaging/test/test_ssl.bats+++ b/solr/packaging/test/test_ssl.bats@@ -572,26 +572,26 @@ teardown() {# Replace server1 keystore with client'scp cert2.keystore.p12 server1.keystore.p12)- # Give some time for the server reload- sleep 6+ # Wait for Jetty keystore-reload scan to pick up the new certificate+ wait_for 30 1 solr api --solr-url "https://localhost:${SOLR_PORT}/solr/admin/info/system"- run solr healthcheck --solr-url https://localhost:${SOLR_PORT}+ run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT}# Server 2 still uses the cert1, so this request should failrun ! solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/test/select?q=query2"- run ! solr healthcheck --solr-url https://localhost:${SOLR2_PORT}+ run ! solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT}(cd "$ssl_dir"# Replace server2 keystore with client'scp cert2.keystore.p12 server2.keystore.p12)- # Give some time for the server reload- sleep 6+ # Wait for Jetty keystore-reload scan to pick up the new certificate+ wait_for 30 1 solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/admin/info/system"- run solr healthcheck --solr-url https://localhost:${SOLR_PORT}- run solr healthcheck --solr-url https://localhost:${SOLR2_PORT}+ run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT}+ run solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT}run solr api --solr-url "https://localhost:${SOLR_PORT}/solr/test/select?q=query3"assert_output --partial '"numFound":0'diff --git a/solr/packaging/test/test_start_solr.bats b/solr/packaging/test/test_start_solr.batsindex 0b206dc7278..f85c0e5918d 100644--- a/solr/packaging/test/test_start_solr.bats+++ b/solr/packaging/test/test_start_solr.bats@@ -25,7 +25,7 @@ teardown() {# save a snapshot of SOLR_HOME for failed testssave_home_on_failure- solr stop --all >/dev/null 2>&1+ SOLR_STOP_WAIT=30 solr stop --all >/dev/null 2>&1}@test "SOLR-11740 check 'solr stop' connection" {diff --git a/solr/packaging/test/test_status.bats b/solr/packaging/test/test_status.batsindex 69a5278aca1..c03e07bfc60 100644--- a/solr/packaging/test/test_status.bats+++ b/solr/packaging/test/test_status.bats@@ -17,40 +17,38 @@load bats_helper-setup() {+setup_file() {common_clean_setup+ solr start+}++teardown_file() {+ common_setup+ solr stop --all+}++setup() {+ common_setup}teardown() {# save a snapshot of SOLR_HOME for failed testssave_home_on_failure-- solr stop --all >/dev/null 2>&1}@test "status detects locally running solr" {- run solr status- assert_output --partial "No Solr nodes are running."- solr startrun solr statusassert_output --partial "running on port ${SOLR_PORT}"- solr stop- run solr status- assert_output --partial "No Solr nodes are running."}@test "status with --solr-url from user" {- solr startrun solr status --solr-url http://localhost:${SOLR_PORT}assert_output --partial "\"solr_home\":"- solr stop}@test "status with --port from user" {- solr startrun solr status --port ${SOLR_PORT}assert_output --partial "running on port ${SOLR_PORT}"- solr stop}@test "multiple connection options are prevented" {@@ -64,8 +62,6 @@ teardown() {}@test "status with --short format" {- solr startrun solr status --port ${SOLR_PORT} --shortassert_output --partial "http://localhost:${SOLR_PORT}/solr"- solr stop}diff --git a/solr/packaging/test/test_zk.bats b/solr/packaging/test/test_zk.batsindex 4cb77a6f590..e2732354799 100644--- a/solr/packaging/test/test_zk.bats+++ b/solr/packaging/test/test_zk.bats@@ -59,13 +59,11 @@ teardown() {}@test "listing out files" {- sleep 1run solr zk ls / -z localhost:${ZK_PORT} --recursiveassert_output --partial "aliases.json"}@test "connecting to solr via various solr urls and zk hosts" {- sleep 1run solr zk ls / --solr-url http://localhost:${SOLR_PORT}assert_output --partial "aliases.json"@@ -90,20 +88,17 @@ teardown() {run solr zk cp myfile.txt zk:/myfile.txt -z localhost:${ZK_PORT}assert_output --partial "Copying from 'myfile.txt' to 'zk:/myfile.txt'. ZooKeeper at localhost:${ZK_PORT}"- sleep 1run solr zk ls / -z localhost:${ZK_PORT}assert_output --partial "myfile.txt"touch myfile2.txtrun solr zk cp myfile2.txt zk:myfile2.txt -z localhost:${ZK_PORT}- sleep 1run solr zk ls / -z localhost:${ZK_PORT}assert_output --partial "myfile2.txt"touch myfile3.txtrun solr zk cp myfile3.txt zk:/myfile3.txt -z localhost:${ZK_PORT}assert_output --partial "Copying from 'myfile3.txt' to 'zk:/myfile3.txt'. ZooKeeper at localhost:${ZK_PORT}"- sleep 1run solr zk ls / -z localhost:${ZK_PORT}assert_output --partial "myfile3.txt"@@ -125,7 +120,7 @@ teardown() {assert_output --partial "Uploading"refute_output --partial "ERROR"- sleep 1+ wait_for 10 1 bash -c "curl -s 'http://localhost:${SOLR_PORT}/api/configsets' | grep -q techproducts2"run curl "http://localhost:${SOLR_PORT}/api/configsets"assert_output --partial '"configSets":["_default","techproducts2"]'}@@ -172,8 +167,6 @@ teardown() {}@test "env var ZK_HOST is honored" {- sleep 1-# Need to unset SOLR_PORT to avoid the tool being smart and look at SOLR_PORTexport SOLR_PORT_KEEP=$SOLR_PORT_LISTENunset SOLR_PORT_LISTEN