def per_exec_ws(folder) { return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder } // initialize source codes def init_git() { retry(5) { checkout scm } // Add more info about job node sh ( script: './tests/scripts/task_show_node_info.sh', label: 'Show executor node info', ) // Determine merge commit to use for all stages if (env.CHANGE_TARGET) { // This is a PR build, so merge with the latest of the PR's target branch // (e.g. main or a release branch like v0.25.0). merge_with_target() } else { // This is a branch build (main or a release branch). Only set // upstream_revision to HEAD and skip merging to avoid a race with another // commit merged to the branch. update_upstream_revision("HEAD") } sh( script: """ set -eux . ${jenkins_scripts_root}/retry.sh retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 """, label: 'Update git submodules', ) checkout_trusted_files() } def update_upstream_revision(git_ref) { if (upstream_revision == null) { upstream_revision = sh( script: "git log -1 ${git_ref} --format=\'%H\'", label: 'Determine upstream revision', returnStdout: true, ).trim() } } def merge_with_target() { def target = env.CHANGE_TARGET sh ( script: "git fetch origin ${target}", label: "Fetch target branch ${target}", ) update_upstream_revision("FETCH_HEAD") sh ( script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}", label: "Merge to origin/${target}" ) } def docker_init(image) { // Clear out all Docker images that aren't going to be used sh( script: """ set -eux docker image ls --all IMAGES=\$(docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %}) echo -e "Found images:\\n\$IMAGES" echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } docker image ls --all """, label: 'Clean old Docker images', ) if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials ecr_pull(image) } else { sh( script: """ set -eux . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${image} """, label: 'Pull docker image', ) } } def ecr_pull(full_name) { aws_account_id = sh( returnStdout: true, script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', label: 'Get AWS ID' ).trim() try { withEnv([ "AWS_ACCOUNT_ID=${aws_account_id}", 'AWS_DEFAULT_REGION={{ aws_default_region }}', "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { sh( script: ''' set -eux aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO ''', label: 'Log in to ECR' ) sh( script: """ set -eux . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${full_name} """, label: 'Pull image from ECR' ) } } finally { withEnv([ "AWS_ACCOUNT_ID=${aws_account_id}", 'AWS_DEFAULT_REGION={{ aws_default_region }}', "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { sh( script: 'docker logout $AWS_ECR_REPO', label: 'Clean up login credentials' ) } } } def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', variable: 'GITHUB_TOKEN', )]) { // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests result = sh ( returnStatus: true, script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", label: 'Check if CI should run slow tests', ) } return result == 0 } def cancel_previous_build() { // cancel previous build if it is not on main. if (env.BRANCH_NAME != 'main') { def buildNumber = env.BUILD_NUMBER as int // Milestone API allows us to cancel previous build // with the same milestone number if (buildNumber > 1) milestone(buildNumber - 1) milestone(buildNumber) } } def is_last_build() { // check whether it is last build try { return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number } catch (Throwable ex) { echo 'Error during check is_last_build ' + ex.toString() return false } } def checkout_trusted_files() { // trust everything from branch builds if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { return; } // trust peoople listed in CONTRIBUTING.md grep_code = sh( returnStatus: true, script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'", label: 'Check if change is from a contributor', ) if (grep_code == 1) { // Any scripts that run on the bare host and not inside a Docker container // (especially those that access secrets) should be checked out here so // only trusted versions are used in CI sh( script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", label: 'Check out trusted files', ) } } def should_skip_ci(pr_number) { if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { // never skip CI on build sourced from a branch return false } glob_skip_ci_code = sh ( returnStatus: true, script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", label: 'Check if CI should be skipped due to changed files', ) if (glob_skip_ci_code == 0) { return true } withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', variable: 'GITHUB_TOKEN', )]) { // Exit code of 1 means run full CI (or the script had an error, so run // full CI just in case). Exit code of 0 means skip CI. git_skip_ci_code = sh ( returnStatus: true, script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", label: 'Check if CI should be skipped', ) } return git_skip_ci_code == 0 } def prepare(node_type) { stage('Prepare') { node(node_type) { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { init_git() if (env.DETERMINE_DOCKER_IMAGES == 'yes') { sh( script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }} {% endfor %}", label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', ) // Pull image names from the results of should_rebuild_docker.py {% for image in images %} {{ image.name }} = sh( script: "cat .docker-image-names/{{ image.name }}", label: "Find docker image name for {{ image.name }}", returnStdout: true, ).trim() {% endfor %} } {% for image in images %} {{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }} {% endfor %} sh (script: """ echo "Docker images being used in this build:" {% for image in images %} echo " {{ image.name }} = ${ {{- image.name -}} }" {% endfor %} """, label: 'Docker image names') is_docs_only_build = sh ( returnStatus: true, script: "./${jenkins_scripts_root}/git_change_docs.sh", label: 'Check for docs only changes', ) skip_ci = should_skip_ci(env.CHANGE_ID) skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) rebuild_docker_images = sh ( returnStatus: true, script: "./${jenkins_scripts_root}/git_change_docker.sh", label: 'Check for any docker changes', ) if (skip_ci) { // Don't rebuild when skipping CI rebuild_docker_images = false } } } } }