Jenkinsfile.crystalball

#!/usr/bin/env groovy

/*
 * Copyright (C) 2021 - present Instructure, Inc.
 *
 * This file is part of Canvas.
 *
 * Canvas is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the Free
 * Software Foundation, version 3 of the License.
 *
 * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
 * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
 * details.
 *
 * You should have received a copy of the GNU Affero General Public License along
 * with this program. If not, see <http://www.gnu.org/licenses/>.
 */

library 'canvas-builds-library'
loadLocalLibrary('local-lib', 'build/new-jenkins/library')

commitMessageFlag.setDefaultValues(commitMessageFlagDefaults() + commitMessageFlagPrivateDefaults())

@groovy.transform.Field
def rspecqNodeTotal = 50

def getMigrationsTag(name) {
  (env.GERRIT_REFSPEC.contains('master')) || !migrations.cacheLoadFailed() ? migrations.imageMergeTag(name) : migrations.imagePatchsetTag(name)
}

def getPatchsetTag() {
  (env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset()
}

def getResultsHTMLUrl() {
  return "${env.BUILD_URL}/artifact/crystalball_map.yml"
}

def sendCrystalballSlack(status) {
  def message = ":alert: <$env.BUILD_URL|Crystalball Map Build> ${status}! :alert:\nResolve this issue to prevent further build failures!"
  slackSend channel: '#crystalball-noisy,#devx-alerts', color: 'danger', message: message
}

def redisUrl() {
  return "redis://${env.TEST_QUEUE_HOST}:6379"
}

def collectCrystalballMap() {
  stage('Collect Crystalball Map') {
    copyArtifacts(
      filter: 'tmp/*/crystalball/**',
      optional: true,
      projectName: env.JOB_NAME,
      selector: specific(env.BUILD_NUMBER),
    )

    sh """
      docker compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/:/tmp \
      -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
      --name crystalball-parser \
      canvas bash -c 'ruby build/new-jenkins/crystalball_merge_coverage.rb "/tmp/*/crystalball/"'
    """

    sh 'docker cp crystalball-parser:/usr/src/app/crystalball_map.yml .'
    archiveArtifacts allowEmptyArchive: true, artifacts: 'crystalball_map.yml'

    def message = "<$env.BUILD_URL/testReport|Latest Crystalball Map Generated> - <${getResultsHTMLUrl()}|Map>\n"
    try {
      def mapSpecInfo = sh(script: """
        docker compose run --rm \
          -v \$(pwd)/\$LOCAL_WORKDIR/crystalball_map.yml/:/usr/src/app/crystalball_map.yml \
          -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \
          -v \$(pwd)/\$LOCAL_WORKDIR/gems/plugins/:/usr/src/app/gems/plugins \
          -v \$(pwd)/\$LOCAL_WORKDIR/spec:/usr/src/app/spec \
          canvas bash -c 'ruby build/new-jenkins/crystalball_map_smoke_test.rb'
        """
      , returnStdout: true)
      message = message + "\n" + mapSpecInfo
      // Only alert and push to s3 on periodic jobs, not ones resulting from manual tests
      if (env.CRYSTALBALL_MAP_PUSH_TO_S3 == '1' && env.GERRIT_EVENT_TYPE != 'comment-added') {
        withCredentials([usernamePassword(credentialsId: 'INSENG_CANVAS_CI_AWS_ACCESS', usernameVariable: 'INSENG_AWS_ACCESS_KEY_ID', passwordVariable: 'INSENG_AWS_SECRET_ACCESS_KEY')]) {
          def awsCreds = "AWS_DEFAULT_REGION=us-west-2 AWS_ACCESS_KEY_ID=${INSENG_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${INSENG_AWS_SECRET_ACCESS_KEY}"
          sh "$awsCreds aws s3 cp crystalball_map.yml s3://instructure-canvas-ci/"
        }
      }
    } catch(e) {
      message = message + "\nMap Invalid!"
    } finally {
      echo message
      slackSend channel: '#crystalball-noisy', message: message
    }
  }
}

env.BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
env.COMPOSE_FILE = 'docker-compose.new-jenkins.yml:docker-compose.new-jenkins-selenium.yml'
env.COMPOSE_PROJECT_NAME = 'crystalball-map'
env.FORCE_FAILURE = commitMessageFlag("force-failure-rspec").asBooleanInteger()
env.RERUNS_RETRY = commitMessageFlag('rspecq-max-requeues').asType(Integer)
env.RSPECQ_FILE_SPLIT_THRESHOLD = '9999'
env.RSPECQ_MAX_REQUEUES = commitMessageFlag('rspecq-max-requeues').asType(Integer)
env.RSPEC_PROCESSES = commitMessageFlag('rspecq-processes').asType(Integer)
env.TEST_PATTERN = '^./(spec|gems/plugins/.*/spec_canvas)/'
env.POSTGRES_PASSWORD = 'sekret'
env.POSTGRES = configuration.postgres()
env.RUBY = configuration.ruby()
env.PATCHSET_TAG = getPatchsetTag()
env.BASE_RUNNER_PREFIX = configuration.buildRegistryPath('base-runner')
env.DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations')
env.KARMA_RUNNER_PREFIX = configuration.buildRegistryPath('karma-runner')
env.LINTERS_RUNNER_PREFIX = configuration.buildRegistryPath('linters-runner')
env.POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations')
env.RUBY_RUNNER_PREFIX = configuration.buildRegistryPath('ruby-runner')
env.YARN_RUNNER_PREFIX = configuration.buildRegistryPath('yarn-runner')
env.WEBPACK_BUILDER_PREFIX = configuration.buildRegistryPath('webpack-builder')
env.WEBPACK_ASSETS_PREFIX = configuration.buildRegistryPath('webpack-assets')
env.IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized()
env.DYNAMODB_IMAGE_TAG = "$env.DYNAMODB_PREFIX:$env.IMAGE_CACHE_MERGE_SCOPE-$env.RSPEC_PROCESSES"
env.POSTGRES_IMAGE_TAG = "$env.POSTGRES_PREFIX:$env.IMAGE_CACHE_MERGE_SCOPE-$env.RSPEC_PROCESSES"
env.POSTGRES_CLIENT = configuration.postgresClient()
env.RSPECQ_REDIS_URL = redisUrl()
env.LOCAL_WORKDIR = pipelineHelpers.getLocalWorkDir()

node(nodeLabel()) {
  timeout(time: 60, unit: 'MINUTES') {
    ansiColor('xterm') {
      timestamps {
        def buildFailed = false
        try {
          stage('Clean Workspace') {
            pipelineHelpers.cleanupWorkspace()
          }

          stage('Checkout Code') {
            sh 'rm -vrf ./tmp'
            checkout scm
            distribution.stashBuildScripts()
          }

          stage('Setup') {
            cleanAndSetup()
            setupStage()
          }

          stage('Build Docker Image') {
            def stageStartTime = System.currentTimeMillis()
            try {
              buildDockerImageStage.patchsetImage()
            } finally {
              buildSummaryReport.trackStage('Build Docker Image', stageStartTime)
            }
          }

          stage('Run Migrations') {
            def stageStartTime = System.currentTimeMillis()
            try {
              runMigrationsStage()
            } finally {
              buildSummaryReport.trackStage('Run Migrations', stageStartTime)
            }
          }

          stage('Run Tests') {
            def rspecqStages = [:]

            stage('Setup Docker Images') {
              rspecStage.setupNode()
            }

            rspecqStages['Reporter'] = {
              try {
                sh(script: "docker run -e SENTRY_DSN -e RSPECQ_REDIS_URL -t $env.PATCHSET_TAG bundle exec rspecq \
                  --build=${env.JOB_NAME}_build${env.BUILD_NUMBER} \
                  --queue-wait-timeout 120 \
                  --redis-url $env.RSPECQ_REDIS_URL \
                  --report", label: 'Reporter')
              } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) {
                if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) {
                  sh '''#!/bin/bash
                    ids=($(docker ps -aq --filter "name=canvas-"))
                    for i in "${ids[@]}"
                      do
                        docker exec $i bash -c "cat /usr/src/app/log/cmd_output/*.log"
                    done
                  '''
                }
                throw e
              }
            }

            // RSpecQ Set 00 runs on main coordinator node
            rspecqStages['RSpecQ 00'] = {
              withEnv([
                "CI_NODE_INDEX=00",
                "BUILD_NAME=${env.JOB_NAME}_build${env.BUILD_NUMBER}",
                "CRYSTALBALL_MAP=1"
              ]) {
                def stageName = 'RSpecQ 00'
                def stageStartTime = System.currentTimeMillis()
                try {
                  stage('00 Run Tests') {
                    catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
                      rspecStage.runRspecqSuite()
                    }
                  }

                  stage('00 Collect Results') {
                    rspecStage.tearDownNode()
                  }
                } finally {
                  buildSummaryReport.trackStage('RSpecQ Set 00', stageStartTime)
                }
              }
            }

            // Remaining nodes (1 through rspecqNodeTotal-1)
            for (int i = 1; i < rspecqNodeTotal; i++) {
              def indexStr = String.format('%02d', i)
              rspecqStages["RSpecQ ${indexStr}"] = {
                rspecStage.runRspecQWorkerNode(indexStr, ["CRYSTALBALL_MAP=1"])
              }
            }

            parallel(rspecqStages)
          }
        } catch (e) {
          // currentBuild.currentResult is not updated until after the finally
          // block completes, so it reads SUCCESS even on failure. Capture the
          // failure here so the finally block can report the correct status.
          buildFailed = true
          throw e
        } finally {
          def buildResult = buildFailed ? 'FAILURE' : currentBuild.currentResult
          pipelineHelpers.cleanupDocker()

          collectCrystalballMap()
          
          buildSummaryReport.addFailureRun('Main Build', currentBuild)
          if (buildFailed || currentBuild.currentResult != 'SUCCESS') {
            sendCrystalballSlack(buildFailed ? 'FAILURE' : currentBuild.currentResult)
          }
          buildSummaryReport.publishReport('Build Summary Report', buildResult)
          buildSummaryReport.saveRunManifest()
        }
      }
    }
  }
}