diff --git a/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java b/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java index e8cb48874b68..9282af37e520 100644 --- a/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PackageToolTest.java @@ -373,6 +373,63 @@ public static void testForResponseElement( success); } + /** Validates that collection existence is checked before package resolution. */ + @Test + public void testDeployValidationMessages() throws Exception { + String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString(); + + withBasicAuth(CollectionAdminRequest.createCollection("validation-test", "conf1", 1, 1)) + .processAndWait(cluster.getSolrClient(), 10); + + CLITestHelper.TestingRuntime captureRuntime = new CLITestHelper.TestingRuntime(true); + PackageTool tool = new PackageTool(captureRuntime); + + // Collection exists but package does not — collection validation should pass, + // package lookup should fail. + tool.runTool( + SolrCLI.processCommandLineArgs( + tool, + new String[] { + "--solr-url", + solrUrl, + "deploy", + "NONEXISTENT_PKG", + "--collections", + "validation-test", + "--credentials", + SecurityJson.USER_PASS + })); + String deployOut = captureRuntime.getOutput(); + assertFalse( + "Should not complain about invalid collection", deployOut.contains("Invalid collection")); + assertTrue( + "Should report missing package", + deployOut.contains("Package instance doesn't exist: NONEXISTENT_PKG:null")); + + captureRuntime.clearOutput(); + + // Undeploy of a package that was never deployed should give a clear message. + tool.runTool( + SolrCLI.processCommandLineArgs( + tool, + new String[] { + "--solr-url", + solrUrl, + "undeploy", + "NONEXISTENT_PKG", + "--collections", + "validation-test", + "--credentials", + SecurityJson.USER_PASS + })); + String undeployOut = captureRuntime.getOutput(); + assertFalse( + "Should not complain about invalid collection", undeployOut.contains("Invalid collection")); + assertTrue( + "Should report package not deployed", + undeployOut.contains("Package NONEXISTENT_PKG not deployed on collection validation-test")); + } + private void run(PackageTool tool, String[] args) throws Exception { int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); assertEquals("Non-zero status returned for: " + Arrays.toString(args), 0, res); diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 7320f2ebe2b2..953adafe01d6 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -281,6 +281,10 @@ task integrationTests(type: BatsTask) { } } + if (!Boolean.parseBoolean((String) project.findProperty('solr.bats.nightly'))) { + filterTags = '!nightly' + } + inputs.dir(distDir) inputs.dir('test') // Track .bats test files and helper as inputs outputs.dir(integrationTestOutput) @@ -320,6 +324,10 @@ class BatsTask extends Exec { @Input var testFiles = [] + @Input + @Optional + String filterTags = null + @Option(option = "tests", description = "Sets test cases to be included") public void setTestNamePatterns(List tests) { // TODO: bats --filter @@ -335,6 +343,9 @@ class BatsTask extends Exec { if (logger.isInfoEnabled()) { batsArgs << '--verbose-run' } + if (filterTags) { + batsArgs += ['--filter-tags', filterTags] + } batsArgs += ['-T', '--print-output-on-failure', '--report-formatter', 'junit', '--output', "$project.buildDir/test-output"] // Note: tests to run must be listed after all other arguments batsArgs += testFiles.empty ? [testDir] : testFiles diff --git a/solr/packaging/test/README.md b/solr/packaging/test/README.md index bd73d4945892..e575502090b0 100644 --- a/solr/packaging/test/README.md +++ b/solr/packaging/test/README.md @@ -55,6 +55,24 @@ Some tests will start clusters or create collections, It is recommended that you install and run `shellcheck` to verify your test scripts and catch common mistakes before committing your changes. +## Nightly Tests + +Some BATS tests are slow or require network access (e.g. downloading models) and are +excluded from regular runs by default. These tests use BATS native tagging via +`# bats test_tags=nightly` and are filtered out with `--filter-tags !nightly`. + +To include nightly tests, pass the Gradle property `solr.bats.nightly`: + + ./gradlew integrationTests -Psolr.bats.nightly=true + +When writing a new test that should only run in nightly builds, add the BATS tag comment +directly before the `@test` declaration: + + # bats test_tags=nightly + @test "my slow test" { + ... + } + ## Limitations 1. Currently this test suite is only available for \*nix environments. Although, newer diff --git a/solr/packaging/test/test_example.bats b/solr/packaging/test/test_example.bats index a15ba4b6f625..a5b4108e549d 100644 --- a/solr/packaging/test/test_example.bats +++ b/solr/packaging/test/test_example.bats @@ -17,25 +17,36 @@ load bats_helper -setup() { +setup_file() { common_clean_setup + solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld" + solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000 + solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000 +} + +teardown_file() { + common_setup + solr stop --all >/dev/null 2>&1 +} + +setup() { + common_setup } teardown() { # save a snapshot of SOLR_HOME for failed tests save_home_on_failure +} - solr stop --all >/dev/null 2>&1 +@test "start -e cloud works with --no-prompt" { + solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000 + solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000 } @test "start -e cloud works with --jvm-opts" { - solr start -e cloud --no-prompt --jvm-opts "-Dcustom.prop=helloworld" - solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 60000 - solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 60000 - run curl "http://localhost:${SOLR_PORT}/solr/admin/info/properties" assert_output --partial 'helloworld' - + run curl "http://localhost:${SOLR2_PORT}/solr/admin/info/properties" assert_output --partial 'helloworld' } diff --git a/solr/packaging/test/test_example_noprompt.bats b/solr/packaging/test/test_example_noprompt.bats deleted file mode 100644 index 19329a8041af..000000000000 --- a/solr/packaging/test/test_example_noprompt.bats +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bats - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load bats_helper - -setup() { - common_clean_setup -} - -teardown() { - # save a snapshot of SOLR_HOME for failed tests - save_home_on_failure - - solr stop --all >/dev/null 2>&1 -} - -@test "start -e cloud works with --no-prompt" { - solr start -e cloud --no-prompt - solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000 - solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000 -} diff --git a/solr/packaging/test/test_healthcheck.bats b/solr/packaging/test/test_healthcheck.bats index e8f6069cb085..5492fb8b5141 100644 --- a/solr/packaging/test/test_healthcheck.bats +++ b/solr/packaging/test/test_healthcheck.bats @@ -29,22 +29,18 @@ teardown() { } @test "healthcheck on cloud solr" { - solr start -e films - run solr healthcheck -c films + solr start + solr create -c healthcheck_test -d _default + # Local + run solr healthcheck -c healthcheck_test refute_output --partial 'error' - -} - -@test "healthcheck on remote cloud solr" { - solr start -e films - run solr healthcheck -c films --solr-connection http://localhost:${SOLR_PORT}/solr + # Remote + run solr healthcheck -c healthcheck_test --solr-connection http://localhost:${SOLR_PORT}/solr refute_output --partial 'error' - } @test "healthcheck errors on standalone solr" { - solr start --user-managed -e films - run solr healthcheck -c films + solr start --user-managed + run solr healthcheck -c healthcheck_test assert_output --partial 'Healthcheck tool only works in Solr Cloud mode' - } diff --git a/solr/packaging/test/test_opennlp.bats b/solr/packaging/test/test_opennlp.bats index 69ee71253cf4..4f71259682f5 100644 --- a/solr/packaging/test/test_opennlp.bats +++ b/solr/packaging/test/test_opennlp.bats @@ -42,8 +42,8 @@ teardown() { # long test as an "integration" test is something we decide is okay? # I also have dreams of incorporating this as code snippets in a Tutorial via the ascii doc tags # like we use for the SolrJ code snippets. That way we know the snippets continue to work! +# bats test_tags=nightly @test "Check lifecycle of sentiment classification" { - echo "Downloading onnx model and vocab..." mkdir -p ${SOLR_TIP}/models/sentiment/ diff --git a/solr/packaging/test/test_packages.bats b/solr/packaging/test/test_packages.bats deleted file mode 100644 index 1f60523e53f0..000000000000 --- a/solr/packaging/test/test_packages.bats +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bats - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load bats_helper - -setup() { - common_clean_setup -} - -teardown() { - # save a snapshot of SOLR_HOME for failed tests - save_home_on_failure - - solr stop --all >/dev/null 2>&1 -} - -@test "lifecycle of package" { - run solr start -Dsolr.packages.enabled=true - - run solr package --help - assert_output --partial "Add a repository to Solr" - - run solr package list-available - assert_output --partial "Available packages:" -} - -@test "deploying and undeploying a collection level package" { - run solr start -Dsolr.packages.enabled=true - - solr create -c foo-1.2 - - # Deploy package - the package doesn't need to exist before the collection validation kicks in - run solr package deploy PACKAGE_NAME --collections foo-1.2 - # assert_output --partial "Deployment successful" - refute_output --partial "Invalid collection" - - # Until PACKAGE_NAME refers to an actual installable package, this is as far as we get. - assert_output --partial "Package instance doesn't exist: PACKAGE_NAME:null" - - # Undeploy package - run solr package undeploy PACKAGE_NAME --collections foo-1.2 - refute_output --partial "Invalid collection" - assert_output --partial "Package PACKAGE_NAME not deployed on collection foo-1.2" -} - -# This test is useful if you are debugging/working with packages. -# We have disabled it for now since it depends on a live internet -# connection to run. This could be updated with a local Repo server if we had -# a package that is part of the Solr project to use. -@test "deploying and undeploying a cluster level package" { - skip "For developing package infra; requires a connection to github" - - run solr start -Dsolr.packages.enabled=true - - run solr package add-repo splainer "https://raw.githubusercontent.com/o19s/splainer/refs/heads/main/solr-splainer-package/repo/" - assert_output --partial "Added repository: splainer" - - run solr package list-available - assert_output --partial "solr-splainer Splainer for Solr" - run solr package install solr-splainer - assert_output --partial "solr-splainer installed." - - run solr package deploy solr-splainer -y --cluster - assert_output --partial "Deployment successful" - - run -0 curl --fail http://localhost:${SOLR_PORT}/v2/splainer/index.html -} diff --git a/solr/packaging/test/test_ssl.bats b/solr/packaging/test/test_ssl.bats index 5075ec763380..cf92d144d4f9 100644 --- a/solr/packaging/test/test_ssl.bats +++ b/solr/packaging/test/test_ssl.bats @@ -572,26 +572,26 @@ teardown() { # Replace server1 keystore with client's cp cert2.keystore.p12 server1.keystore.p12 ) - # Give some time for the server reload - sleep 6 + # Wait for Jetty keystore-reload scan to pick up the new certificate + wait_for 30 1 solr api --solr-url "https://localhost:${SOLR_PORT}/solr/admin/info/system" - run solr healthcheck --solr-url https://localhost:${SOLR_PORT} + run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT} # Server 2 still uses the cert1, so this request should fail run ! solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/test/select?q=query2" - run ! solr healthcheck --solr-url https://localhost:${SOLR2_PORT} + run ! solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT} ( cd "$ssl_dir" # Replace server2 keystore with client's cp cert2.keystore.p12 server2.keystore.p12 ) - # Give some time for the server reload - sleep 6 + # Wait for Jetty keystore-reload scan to pick up the new certificate + wait_for 30 1 solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/admin/info/system" - run solr healthcheck --solr-url https://localhost:${SOLR_PORT} - run solr healthcheck --solr-url https://localhost:${SOLR2_PORT} + run solr healthcheck -c test --solr-url https://localhost:${SOLR_PORT} + run solr healthcheck -c test --solr-url https://localhost:${SOLR2_PORT} run solr api --solr-url "https://localhost:${SOLR_PORT}/solr/test/select?q=query3" assert_output --partial '"numFound":0' diff --git a/solr/packaging/test/test_start_solr.bats b/solr/packaging/test/test_start_solr.bats index 0b206dc72786..f85c0e5918de 100644 --- a/solr/packaging/test/test_start_solr.bats +++ b/solr/packaging/test/test_start_solr.bats @@ -25,7 +25,7 @@ teardown() { # save a snapshot of SOLR_HOME for failed tests save_home_on_failure - solr stop --all >/dev/null 2>&1 + SOLR_STOP_WAIT=30 solr stop --all >/dev/null 2>&1 } @test "SOLR-11740 check 'solr stop' connection" { diff --git a/solr/packaging/test/test_status.bats b/solr/packaging/test/test_status.bats index 69a5278aca18..c03e07bfc60a 100644 --- a/solr/packaging/test/test_status.bats +++ b/solr/packaging/test/test_status.bats @@ -17,40 +17,38 @@ load bats_helper -setup() { +setup_file() { common_clean_setup + solr start +} + +teardown_file() { + common_setup + solr stop --all +} + +setup() { + common_setup } teardown() { # save a snapshot of SOLR_HOME for failed tests save_home_on_failure - - solr stop --all >/dev/null 2>&1 } @test "status detects locally running solr" { - run solr status - assert_output --partial "No Solr nodes are running." - solr start run solr status assert_output --partial "running on port ${SOLR_PORT}" - solr stop - run solr status - assert_output --partial "No Solr nodes are running." } @test "status with --solr-url from user" { - solr start run solr status --solr-url http://localhost:${SOLR_PORT} assert_output --partial "\"solr_home\":" - solr stop } @test "status with --port from user" { - solr start run solr status --port ${SOLR_PORT} assert_output --partial "running on port ${SOLR_PORT}" - solr stop } @test "multiple connection options are prevented" { @@ -64,8 +62,6 @@ teardown() { } @test "status with --short format" { - solr start run solr status --port ${SOLR_PORT} --short assert_output --partial "http://localhost:${SOLR_PORT}/solr" - solr stop } diff --git a/solr/packaging/test/test_zk.bats b/solr/packaging/test/test_zk.bats index 4cb77a6f5902..e27323547992 100644 --- a/solr/packaging/test/test_zk.bats +++ b/solr/packaging/test/test_zk.bats @@ -59,13 +59,11 @@ teardown() { } @test "listing out files" { - sleep 1 run solr zk ls / -z localhost:${ZK_PORT} --recursive assert_output --partial "aliases.json" } @test "connecting to solr via various solr urls and zk hosts" { - sleep 1 run solr zk ls / --solr-url http://localhost:${SOLR_PORT} assert_output --partial "aliases.json" @@ -90,20 +88,17 @@ teardown() { run solr zk cp myfile.txt zk:/myfile.txt -z localhost:${ZK_PORT} assert_output --partial "Copying from 'myfile.txt' to 'zk:/myfile.txt'. ZooKeeper at localhost:${ZK_PORT}" - sleep 1 run solr zk ls / -z localhost:${ZK_PORT} assert_output --partial "myfile.txt" touch myfile2.txt run solr zk cp myfile2.txt zk:myfile2.txt -z localhost:${ZK_PORT} - sleep 1 run solr zk ls / -z localhost:${ZK_PORT} assert_output --partial "myfile2.txt" touch myfile3.txt run solr zk cp myfile3.txt zk:/myfile3.txt -z localhost:${ZK_PORT} assert_output --partial "Copying from 'myfile3.txt' to 'zk:/myfile3.txt'. ZooKeeper at localhost:${ZK_PORT}" - sleep 1 run solr zk ls / -z localhost:${ZK_PORT} assert_output --partial "myfile3.txt" @@ -125,7 +120,7 @@ teardown() { assert_output --partial "Uploading" refute_output --partial "ERROR" - sleep 1 + wait_for 10 1 bash -c "curl -s 'http://localhost:${SOLR_PORT}/api/configsets' | grep -q techproducts2" run curl "http://localhost:${SOLR_PORT}/api/configsets" assert_output --partial '"configSets":["_default","techproducts2"]' } @@ -172,8 +167,6 @@ teardown() { } @test "env var ZK_HOST is honored" { - sleep 1 - # Need to unset SOLR_PORT to avoid the tool being smart and look at SOLR_PORT export SOLR_PORT_KEEP=$SOLR_PORT_LISTEN unset SOLR_PORT_LISTEN