lib = library(identifier: 'jenkins@5.4.1', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) pipeline { options { timeout(time: 4, unit: 'HOURS') } agent none environment { AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v3' // required for rpm to create digest sha256 correctly with rpm 4.12+ IMAGE_DEB = 'opensearchstaging/ci-runner:ci-runner-ubuntu2004-opensearch-build-v2' // required for deb to create pkg using debmake/debuild/debhelper } parameters { string( name: 'COMPONENT_NAME', description: 'If this field contains one or more component names (e.g. OpenSearch common-utils ...), will build with "--component ...", else build everything in the INPUT_MANIFEST.', trim: true ) string( name: 'INPUT_MANIFEST', description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', trim: true ) string( name: 'TEST_MANIFEST', description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.', trim: true ) string( name: 'INTEG_TEST_JOB_NAME', description: "Name of integration test job that will be triggered, e.g. Playground/integ-test. A non-null empty value here will skip integration tests.", defaultValue: "integ-test", trim: true ) string( name: 'BWC_TEST_JOB_NAME', description: "Name of backwards compatibility test job that will be triggered, e.g. Playground/bwc-test. A non-null empty value here will skip BWC tests.", defaultValue: "bwc-test", trim: true ) string( // Note: need to update 'verify-parameters' entries if you add new platform(s) name: 'BUILD_PLATFORM', description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)", defaultValue: 'linux macos windows', trim: true ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', choices: ['build_docker', 'build_docker_with_build_number_tag', 'do_not_build_docker'], ) booleanParam( name: 'UPDATE_LATEST_URL', description: 'Update latest url so /latest/ is pointed to this build', defaultValue: true ) booleanParam( name: 'PUBLISH_NOTIFICATION', description: 'Publish the status of this build job or not.', defaultValue: true ) booleanParam( name: 'CREATE_GITHUB_ISSUE', description: 'To create a github issue for failing component or not.', defaultValue: true ) } stages { stage('verify-parameters') { agent { docker { label AGENT_X64 image 'docker/library/alpine:3' registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { echo("Detect Docker Images and Related Parameters") dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST echo("Verify Build Platforms") def build_platform_array = params.BUILD_PLATFORM.tokenize(' ') echo("User Entry Platforms: '${params.BUILD_PLATFORM}', ${build_platform_array}") def all_platforms = "linux macos windows" echo("All Supported Platforms: '${all_platforms}'") if (params.BUILD_PLATFORM == null || params.BUILD_PLATFORM == '') { currentBuild.result = 'ABORTED' error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") } for (String plat : build_platform_array) { if (! all_platforms.contains(plat)) { currentBuild.result = 'ABORTED' error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") } } } } } stage('Initialize the job info yaml report') { agent { docker { label AGENT_X64 image dockerAgent.image registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildInfoYaml( componentName: COMPONENT_NAME, inputManifest: "manifests/$INPUT_MANIFEST", status: "NOT_STARTED", stage: "INITIALIZE_STAGE" ) } } post { always { postCleanup() } } } stage('build') { parallel { stage('build-opensearch-snapshot-linux-x64-tar') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } environment { SNAPSHOT_REPO_URL = "https://aws.oss.sonatype.org/content/repositories/snapshots/" } agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildManifest( componentName: "OpenSearch", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: 'tar', snapshot: true ) echo("Uploading min snapshots to S3") uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' ) } } post { always { postCleanup() } } } stage('build-opensearch-snapshot-linux-arm64-tar') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { docker { label AGENT_ARM64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildManifest( componentName: "OpenSearch", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: 'tar', snapshot: true ) echo("Uploading min snapshots to S3") uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' ) } } post { always { postCleanup() } } } stage('build-opensearch-snapshot-macos-x64-tar') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('macos') } } agent { node { label 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host' } } tools { jdk dockerAgent.javaVersion } steps { script { buildManifest( componentName: "OpenSearch", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'darwin', architecture: 'x64', distribution: 'tar', snapshot: true ) echo("Uploading darwin min snapshots to S3") uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'tar' ) } } post { always { postCleanup() } } } stage('build-opensearch-snapshot-windows-x64-zip') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('windows') } } agent { node { label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' } } tools { jdk dockerAgent.javaVersion } steps { script { buildManifest( componentName: "OpenSearch", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'windows', architecture: 'x64', distribution: 'zip', snapshot: true ) echo("Uploading windows min snapshots to S3") uploadMinSnapshotsToS3( fileActions: [createSha512Checksums()], distribution: 'zip' ) } } post { always { postCleanup() } } } stage('build-and-test-linux-x64-tar') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: "tar" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, x64, tar): ${artifactUrl}" parallel([ 'integ-test': { Boolean skipIntegTests = INTEG_TEST_JOB_NAME == '' echo "${skipIntegTests ? 'Skipping integration tests' : 'Running integration tests'}" if (!skipIntegTests) { def integTestResults = build job: INTEG_TEST_JOB_NAME, propagate: false, wait: false, parameters: [ string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl) ] } }, 'bwc-test': { Boolean skipBwcTests = BWC_TEST_JOB_NAME == '' echo "${skipBwcTests ? 'Skipping BWC tests' : 'Running BWC tests'}" if (!skipBwcTests) { def bwcTestResults = build job: BWC_TEST_JOB_NAME, propagate: false, wait: false, parameters: [ string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_X64) ] } } ]) } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } stage('build-and-test-linux-x64-rpm') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { label AGENT_X64 } stages { stage('build-archive-linux-x64-rpm') { agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildArchive( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: 'rpm', stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } } post { always { postCleanup() } } } stage('assemble-archive-and-test-linux-x64-rpm') { agent { docker { label AGENT_X64 image IMAGE_RPM registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: 'rpm', stashName: "build-archive-linux-x64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, x64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) echo "Trigger rpm validation for linux x64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, wait: false, parameters: [ string(name: 'BUNDLE_MANIFEST_URL', value: bundleManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_X64) ] } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } } } stage('build-and-test-linux-x64-deb') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { label AGENT_X64 } stages { stage('build-archive-linux-x64-deb') { agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildArchive( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: 'deb', stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } } post { always { postCleanup() } } } stage('assemble-archive-and-test-linux-x64-deb') { agent { docker { label AGENT_X64 image IMAGE_DEB registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'x64', distribution: 'deb', stashName: "build-archive-linux-x64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_X64_DEB = artifactUrl echo "buildManifestUrl (linux, x64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, x64, deb): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } } } stage('build-and-test-linux-arm64-tar') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { docker { label AGENT_ARM64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: "tar" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, tar): ${artifactUrl}" parallel([ 'integ-test': { Boolean skipIntegTests = INTEG_TEST_JOB_NAME == '' echo "${skipIntegTests ? 'Skipping integration tests' : 'Running integration tests'}" if (!skipIntegTests) { def integTestResults = build job: INTEG_TEST_JOB_NAME, propagate: false, wait: false, parameters: [ string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl) ] } }, 'bwc-test': { Boolean skipBwcTests = BWC_TEST_JOB_NAME == '' echo "${skipBwcTests ? 'Skipping BWC tests' : 'Running BWC tests'}" if (!skipBwcTests) { def bwcTestResults = build job: BWC_TEST_JOB_NAME, propagate: false, wait: false, parameters: [ string(name: 'TEST_MANIFEST', value: TEST_MANIFEST), string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] } } ]) } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } stage('build-and-test-linux-arm64-rpm') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { label AGENT_X64 } stages { stage('build-archive-linux-arm64-rpm') { agent { docker { label AGENT_ARM64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildArchive( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: 'rpm', stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) } } post { always { postCleanup() } } } stage('assemble-archive-and-test-linux-arm64-rpm') { agent { docker { label AGENT_ARM64 image IMAGE_RPM registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: 'rpm', stashName: "build-archive-linux-arm64-rpm-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) echo "Trigger rpm validation for linux arm64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, wait: false, parameters: [ string(name: 'BUNDLE_MANIFEST_URL', value: bundleManifestUrl), string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } } } stage('build-and-test-linux-arm64-deb') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('linux') } } agent { label AGENT_ARM64 } stages { stage('build-archive-linux-arm64-deb') { agent { docker { label AGENT_ARM64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { buildArchive( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: 'deb', stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) } } post { always { postCleanup() } } } stage('assemble-archive-and-test-linux-arm64-deb') { agent { docker { label AGENT_ARM64 image IMAGE_DEB registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { def buildManifestObj = archiveAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'linux', architecture: 'arm64', distribution: 'deb', stashName: "build-archive-linux-arm64-deb-${JOB_NAME}-${BUILD_NUMBER}" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_ARM64_DEB = artifactUrl echo "buildManifestUrl (linux, arm64, deb): ${buildManifestUrl}" echo "artifactUrl (linux, arm64, deb): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } } } stage('build-and-test-windows-x64-zip') { when { beforeAgent true expression{ params.BUILD_PLATFORM.contains('windows') } } agent { node { label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' } } tools { jdk dockerAgent.javaVersion } steps { script { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", platform: 'windows', architecture: 'x64', distribution: "zip" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" echo "artifactUrl (windows, x64, zip): ${artifactUrl}" } } post { always { script { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) ) postCleanup() } } } } } } stage('update index file') { agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { if (params.UPDATE_LATEST_URL) { uploadIndexFile( indexFilePath: env.INDEX_FILE_PATH ) } } } } stage('docker build') { when { beforeAgent true allOf{ expression { params.BUILD_DOCKER != 'do_not_build_docker' } expression{ params.BUILD_PLATFORM.contains('linux') } } } agent { docker { label AGENT_X64 image dockerAgent.image args dockerAgent.args registryUrl 'https://public.ecr.aws/' alwaysPull true } } steps { script { echo "env.ARTIFACT_URL_LINUX_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" echo "env.ARTIFACT_URL_LINUX_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" buildDockerImage( inputManifest: "manifests/${INPUT_MANIFEST}", buildNumber: "${BUILD_NUMBER}", buildOption: "${BUILD_DOCKER}", artifactUrlX64: env.ARTIFACT_URL_LINUX_X64_TAR, artifactUrlArm64: env.ARTIFACT_URL_LINUX_ARM64_TAR ) } } } } post { always { node(AGENT_X64) { checkout scm script { buildInfoYaml( componentName: COMPONENT_NAME, status: currentBuild.result, stage: "FINALIZE_STAGE" ) unstash "buildInfo_yml" archiveArtifacts artifacts: 'buildInfo.yml' closeBuildSuccessGithubIssue( message: buildMessage(search: 'Successfully built'), search: "Successfully built", inputManifestPath: "manifests/$INPUT_MANIFEST" ) postCleanup() } } } success { node(AGENT_X64) { script { List stages = [] if (params.BUILD_PLATFORM.contains('linux')) { stages = [ 'build-and-test-linux-x64-tar', 'assemble-archive-and-test-linux-x64-rpm', 'assemble-archive-and-test-linux-x64-deb', 'build-and-test-linux-arm64-tar', 'assemble-archive-and-test-linux-arm64-rpm', 'assemble-archive-and-test-linux-arm64-deb', ] } if (params.BUILD_PLATFORM.contains('windows')){ stages.add('build-and-test-windows-x64-zip') } if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get(stages) publishNotification( icon: ':white_check_mark:', message: 'Successful Build', extra: stashed, credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } postCleanup() } } } failure { node(AGENT_X64) { checkout scm script { if (params.PUBLISH_NOTIFICATION) { publishNotification( icon: ':warning:', message: buildMessage(search: 'Error building'), credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } if (params.CREATE_GITHUB_ISSUE) { createBuildFailureGithubIssue( message: buildMessage(search: 'Error building'), inputManifestPath: "manifests/$INPUT_MANIFEST" ) } postCleanup() } } } } }