DAOS-18348 build: Support running Bullseye

vars/buildRpmPost.groovy

@@ -49,8 +49,9 @@
    *
    * config['rpmlint']             Whether to run rpmlint on resulting RPMs.
    *                               Default false.
-   * config['new_rpm']             Whether we are using new RPM or not
-   *                               Default false
+   *
+   * config['productArtifacts']    List of product names and artifact directories to publish.
+   *                               Default is to publish the whole target directory.
    *
    * config['unsuccessful_script'] Script to run if build is not successful.
    *                               Default 'ci/rpm/build_unsuccessful.sh'
@@ -90,23 +91,30 @@ void call(Map config = [:]) {
         }
 
         String product = config.get('product', 'daos-stack')
-        String artdir = 'artifacts/' + target
-        if (config.get('new_rpm', false)) {
-            String deps_dir = 'artifacts/' + target + '/deps'
-            if (fileExists(deps_dir)) {
-                publishToRepository product: 'deps',
+        Map<String, String> productArtifacts = [:]
+        for (String name : config.get('productArtifacts', [])) {
+            if (name == 'daos') {
+                // Use the product name for daos
+                productArtifacts[product] = "artifacts/${target}/${name}"
+            } else {
+                // Use the specified name or other products, e.g. deps
+                productArtifacts[name] = "artifacts/${target}/${name}"
+            }
+        }
+        if (!productArtifacts) {
+            // Publish the whole target directory if no productArtifacts are specified
+            productArtifacts[product] = "artifacts/${target}"
+        }
+        for (String key in productArtifacts.keySet()) {
+            String artifactDir = productArtifacts[key]
+            if (fileExists(artifactDir)) {
+                publishToRepository product: key,
                                     format: repo_format,
                                     maturity: 'stable',
                                     tech: target,
-                                    repo_dir: deps_dir
+                                    repo_dir: artifactDir
             }
-            artdir = 'artifacts/' + target + '/daos'
         }
-        publishToRepository product: product,
-                            format: repo_format,
-                            maturity: 'stable',
-                            tech: target,
-                            repo_dir: artdir
 
         if (config.get('rpmlint', false)) {
             rpmlintMockResults(sh(label: 'Get chroot name',

vars/functionalTest.groovy

@@ -59,13 +59,22 @@
    *
    * config['ftest_arg']         Functional test launch.py arguments.
    *                             Default determined by parseStageInfo().
+   *
+   * config['coverage_stash']    Name to stash coverage artifacts.
+   *                             Default is empty string, which will result in no stashing.
+   *
+   * config['bullseye']          Set to true to use bullseye-sepecific repo.
+   *                             Default false.
+   *
    */
 
 Map call(Map config = [:]) {
     long startDate = System.currentTimeMillis()
     String nodelist = config.get('NODELIST', env.NODELIST)
     String context = config.get('context', 'test/' + env.STAGE_NAME)
     String description = config.get('description', env.STAGE_NAME)
+    String coverage_stash = config.get('coverage_stash', '')
+    Boolean bullseye = config.get('bullseye', false)
 
     Map stage_info = parseStageInfo(config)
 
@@ -93,7 +102,8 @@ Map call(Map config = [:]) {
                  node_count: stage_info['node_count'],
                  distro: image_version,
                  inst_repos: config.get('inst_repos', ''),
-                 inst_rpms: stage_inst_rpms)
+                 inst_rpms: stage_inst_rpms,
+                 bullseye: bullseye)
 
     List stashes = []
     if (config['stashes']) {
@@ -113,6 +123,7 @@ Map call(Map config = [:]) {
     run_test_config['ftest_arg'] = config.get('ftest_arg', stage_info['ftest_arg'])
     run_test_config['context'] = context
     run_test_config['description'] = description
+    run_test_config['coverage_stash'] = coverage_stash
 
     Map runtestData = [:]
     if (config.get('test_function', 'runTestFunctional') ==

vars/getAdditionalPackages.groovy

@@ -0,0 +1,25 @@
+// vars/getAdditionalPackages.groovy
+
+/**
+ *
+ * getAdditionalPackages.groovy
+ *
+ * Get the additional packages for the functional test stages based on the provider and
+ * whether or not bullseye reporting is enabled.
+ *
+ * @ param ucx              whether or not to include UCX packages
+ * @ param bullseye         whether or not the packages are bullseye versioned
+ * @ return a String of space-separated package names
+ */
+String call(Boolean ucx=false, Boolean bullseye=false) {
+    String packages = ''
+    if (ucx) {
+        packages += ' mercury-ucx'
+    } else {
+        packages += ' mercury-libfabric'
+    }
+    if (bullseye) {
+        packages += ' bullseye'
+    }
+    return packages.trim()
+}

vars/getFunctionalPackages.groovy

@@ -0,0 +1,42 @@
+// vars/getFunctionalPackages.groovy
+
+/**
+ * getFunctionalPackages.groovy
+ *
+ * Get the packages to install in the functional test satge.
+ *
+ * @param daosPackages      daos packages to install (with a version)
+ * @param otherPackages     space-separated string of additional non-daos packages to install
+ * @param bullseye          option to include the '.bullseye' extension to the daos package version
+ * @return a scripted stage to run in a pipeline
+ */
+
+String call(String otherPackages, Boolean bullseye=false) {
+    return getFunctionalPackages(null, otherPackages, bullseye)
+}
+
+String call(Boolean ucx=false, Boolean bullseye=false) {
+    String otherPackages = getAdditionalPackages(ucx, bullseye)
+    return getFunctionalPackages(null, otherPackages, bullseye)
+}
+
+String call(String daosPackages, String otherPackages, Boolean bullseye=false) {
+    String packages = ''
+
+    if (daosPackages) {
+        packages += daosPackages
+    } else if (bullseye) {
+        packages += 'daos-bullseye{,-{client,tests,server,serialize,tests-internal}}'
+    } else {
+        packages += 'daos{,-{client,tests,server,serialize,tests-internal}}'
+    }
+
+    // Add non-daos packages
+    if (otherPackages) {
+        packages += " ${otherPackages}"
+    }
+
+    println("getFunctionalPackages(${daosPackages}, ${otherPackages}, ${bullseye}) => ${packages}")
+
+    return packages
+}

vars/getFunctionalTestStage.groovy

@@ -12,7 +12,6 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *      name            functional test stage name
  *      pragma_suffix   functional test stage commit pragma suffix, e.g. '-hw-medium'
  *      label           functional test stage default cluster label
- *      next_version    next daos package version
  *      stage_tags      functional test stage tags always used and combined with all other tags
  *      default_tags    launch.py tags argument to use when no parameter or commit pragma exist
  *      nvme            launch.py --nvme argument to use
@@ -21,16 +20,19 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
  *      distro          functional test stage distro (VM)
  *      image_version   image version to use for provisioning, e.g. el8.8, leap15.6, etc.
  *      base_branch     if specified, checkout sources from this branch before running tests
+ *      other_packages  space-separated string of additional RPM packages to install
+ *      inst_rpms       space-separated string of RPM packages to install on the test nodes;
+ *                          exclusive of other_packages.
  *      run_if_pr       whether or not the stage should run for PR builds
  *      run_if_landing  whether or not the stage should run for landing builds
  *      job_status      Map of status for each stage in the job/build
+ *      coverage_stash  name of stash to include code coverage results from the tests
  * @return a scripted stage to run in a pipeline
  */
 Map call(Map kwargs = [:]) {
     String name = kwargs.get('name', 'Unknown Functional Test Stage')
     String pragma_suffix = kwargs.get('pragma_suffix')
     String label = kwargs.get('label')
-    String next_version = kwargs.get('next_version', null)
     String stage_tags = kwargs.get('stage_tags')
     String default_tags = kwargs.get('default_tags')
     String nvme = kwargs.get('nvme')
@@ -39,10 +41,12 @@ Map call(Map kwargs = [:]) {
     String distro = kwargs.get('distro')
     String image_version = kwargs.get('image_version', null)
     String base_branch = kwargs.get('base_branch')
-    String other_packages = kwargs.get('other_packages', '')
+    String instRpms = kwargs.get(
+        'inst_rpms', getFunctionalPackages(kwargs.get('other_packages', '')))
     Boolean run_if_pr = kwargs.get('run_if_pr', false)
     Boolean run_if_landing = kwargs.get('run_if_landing', false)
     Map job_status = kwargs.get('job_status', [:])
+    String coverage_stash = kwargs.get('coverage_stash', '')
 
     return {
         stage("${name}") {
@@ -83,14 +87,15 @@ Map call(Map kwargs = [:]) {
                             functionalTest(
                                 image_version: image_version,
                                 inst_repos: daosRepos(distro),
-                                inst_rpms: functionalPackages(1, next_version, 'tests-internal') + ' ' + other_packages,
+                                inst_rpms: instRpms,
                                 test_tag: tags,
                                 ftest_arg: getFunctionalArgs(
                                     pragma_suffix: pragma_suffix,
                                     nvme: nvme,
                                     default_nvme: default_nvme,
                                     provider: provider)['ftest_arg'],
-                                test_function: 'runTestFunctionalV2'))
+                                test_function: 'runTestFunctionalV2',
+                                coverage_stash: coverage_stash))
                     } finally {
                         println("[${name}] Running functionalTestPostV2()")
                         functionalTestPostV2()

vars/provisionNodes.groovy

@@ -26,6 +26,8 @@
  * config['timeout']    Timeout in minutes.  Default 30.
  * config['inst_repos'] DAOS stack repos that should be configured.
  * config['inst_rpms']  DAOS stack RPMs that should be installed.
+ * config['bullseye']   Set to true to use bullseye-specific repo.  Default false.
+ *
  *  if timeout is <= 0, then will not wait for provisioning.
  *  if power_only is specified, the nodes will be rebooted and the
  *  provisioning information ignored.
@@ -182,6 +184,7 @@ Map call(Map config = [:]) {
                       'CI_RPM_TEST_VERSION="' + (params.CI_RPM_TEST_VERSION ?: '') + '" ' +
                       'CI_PR_REPOS="' + (params.CI_PR_REPOS ?: '') + '" ' +
                       'HTTPS_PROXY="' + https_proxy + '" ' +
+                      'CI_BULLSEYE="' + (config.get('bullseye', false) ? 'true' : 'false') + '" ' +
                       'ci/provisioning/post_provision_config.sh'
     new_config['post_restore'] = provision_script
     try {

vars/runTestFunctionalV2.groovy

@@ -88,14 +88,21 @@ Map call(Map config = [:]) {
     // Restore the ignore failure setting
     config['ignore_failure'] = ignore_failure
 
-    String coverageFile = 'test.cov'
-    if (!fileExists('test.cov')) {
-        coverageFile += '_not_done'
-        fileOperations([fileCreateOperation(fileName: coverageFile,
-                                            fileContent: '')])
+    // Stash code coverage file if it exists
+    String coverage_stash = config.get('coverage_stash', '')
+    String coverage_dir = "${basedir}bullseye_coverage_logs"
+    String coverage_file = "${coverage_dir}/test.cov"
+    if (coverage_stash) {
+        if (fileExists(coverage_file)) {
+            // String name = 'func' + stage_info['pragma_suffix'] + '-cov'
+            println("[${env.STAGE_NAME}] Stashing ${coverage_file} in ${coverage_stash}")
+            dir(coverage_dir) {
+                stash name: coverage_stash, includes: 'test.cov'
+            }
+        } else {
+            println("[${env.STAGE_NAME}] No ${coverage_file} file found for ${coverage_stash}!")
+        }
     }
-    String name = 'func' + stage_info['pragma_suffix'] + '-cov'
-    stash name: config.get('coverage_stash', name),
-          includes: coverageFile
+
     return runData
 }