lib = library(identifier: 'jenkins@1.0.4', retriever: modernSCM([
    $class: 'GitSCMSource',
    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
]))

pipeline {
    agent none
    options {
        timeout(time: 14, unit: 'DAYS')
    }
    environment {
        AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Docker-Host-Perf-Test'
        AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v3'
        BUNDLE_MANIFEST = 'bundle-manifest.yml'
        JOB_NAME = 'perf-test'
    }
    triggers {
        parameterizedCron '''
            H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 12 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.11/8029/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2

            H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.7.0/7771/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2

            H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 8 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2

            H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 4 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
            H 4 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2
        '''
    }
    parameters {
        string(
            name: 'BUNDLE_MANIFEST_URL',
            description: 'The distribution manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.3.0/6039/linux/x64/tar/dist/opensearch/manifest.yml',
            trim: true
        )
        string(
            defaultValue: 'nyc_taxis',
            name: 'TEST_WORKLOAD',
            description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).',
            trim: true
        )
        string(
            defaultValue: '1',
            name: 'TEST_ITERATIONS',
            description: 'Number of times to run a workload for Mensor (internal client).',
            trim: true
        )
        string(
            defaultValue: '0',
            name: 'WARMUP_ITERATIONS',
            description: 'Number of times to run a workload before collecting data for Mensor (internal client).',
            trim: true
        )
    }

    stages {
        stage('validate-and-set-parameters') {
            agent {
                docker {
                    label AGENT_LABEL
                    image AGENT_IMAGE
                    registryUrl 'https://public.ecr.aws/'
                    alwaysPull true
                }
            }
            steps {
                script {
                    if (BUNDLE_MANIFEST_URL == '') {
                        currentBuild.result = 'ABORTED'
                        error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.")
                    }
                    if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) {
                        currentBuild.result = 'ABORTED'
                        error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.")
                    }
                    if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) {
                        currentBuild.result = 'ABORTED'
                        error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.")
                    }
                    def bundleManifestObj = downloadBuildManifest(
                        url: BUNDLE_MANIFEST_URL,
                        path: BUNDLE_MANIFEST
                    )
                    String buildId = bundleManifestObj.getArtifactBuildId()
                    env.BUILD_ID = buildId
                    env.HAS_SECURITY = bundleManifestObj.components.containsKey("security")
                    env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture()
                    echo "HAS_SECURITY: ${env.HAS_SECURITY}"
                    lib.jenkins.Messages.new(this).add(JOB_NAME, "Performance tests for #${BUILD_ID}")
                    currentBuild.description = "Running performance test for build number: ${BUILD_ID} Manifest: ${BUNDLE_MANIFEST_URL}"
                }
            }
        }
        stage('perf-test') {
            parallel {
                stage('test-with-security') {
                    agent {
                        docker {
                            label AGENT_LABEL
                            image AGENT_IMAGE
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    when {
                        expression { return env.HAS_SECURITY.toBoolean() }
                    }
                    steps {
                        script {
                            def bundleManifestObj = downloadBuildManifest(
                                url: BUNDLE_MANIFEST_URL,
                                path: BUNDLE_MANIFEST
                            )
                            echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
                            echo "BUILD_ID: ${BUILD_ID}"
                            echo "Architecture: ${ARCHITECTURE}"

                            runPerfTestScript(bundleManifest: BUNDLE_MANIFEST,
                                buildId: BUILD_ID,
                                architecture: ARCHITECTURE,
                                insecure: false,
                                workload: TEST_WORKLOAD,
                                testIterations: TEST_ITERATIONS,
                                warmupIterations: WARMUP_ITERATIONS,
                                stackNameSuffix: env.BUILD_NUMBER)

                            lib.jenkins.Messages.new(this).add(JOB_NAME,
                                lib.jenkins.Messages.new(this).get([JOB_NAME]) +
                                "\nPerformance tests with security for ${BUILD_ID} completed")
                        }
                    }
                    post {
                        success {
                            script {
                                uploadTestResults(
                                    buildManifestFileName: BUNDLE_MANIFEST,
                                    jobName: JOB_NAME,
                                    buildNumber: env.BUILD_NUMBER
                                )
                            }
                            postCleanup()
                        }
                        failure {
                            postCleanup()
                        }
                    }
                }
                stage('test-without-security') {
                    agent {
                        docker {
                            label AGENT_LABEL
                            image AGENT_IMAGE
                            registryUrl 'https://public.ecr.aws/'
                            alwaysPull true
                        }
                    }
                    steps {
                        script {
                            def bundleManifestObj = downloadBuildManifest(
                                url: BUNDLE_MANIFEST_URL,
                                path: BUNDLE_MANIFEST
                            )

                            echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
                            echo "BUILD_ID: ${BUILD_ID}"
                            echo "Architecture: ${ARCHITECTURE}"

                            runPerfTestScript(bundleManifest: BUNDLE_MANIFEST,
                                buildId: BUILD_ID,
                                architecture: ARCHITECTURE,
                                insecure: true,
                                workload: TEST_WORKLOAD,
                                testIterations: TEST_ITERATIONS,
                                warmupIterations: WARMUP_ITERATIONS,
                                stackNameSuffix: env.BUILD_NUMBER)

                            lib.jenkins.Messages.new(this).add(JOB_NAME,
                                lib.jenkins.Messages.new(this).get([JOB_NAME]) +
                                "\nPerformance tests without security for ${BUILD_ID} completed")
                        }
                    }
                    post {
                        success {
                            script {
                                uploadTestResults(
                                    buildManifestFileName: BUNDLE_MANIFEST,
                                    jobName: JOB_NAME,
                                    buildNumber: env.BUILD_NUMBER
                                )
                            }
                            postCleanup()
                        }
                        failure {
                            postCleanup()
                        }
                    }
                }
            }
        }
    }

    post {
        success {
            node(AGENT_LABEL) {
                script {
                    def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME])
                    publishNotification(
                        icon: ':white_check_mark:',
                        message: 'Performance Tests Successful',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                    )
                    postCleanup()
                }
            }
        }
        failure {
            node(AGENT_LABEL) {
                script {
                    def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME])
                    publishNotification(
                        icon: ':warning:',
                        message: 'Failed Performance Tests',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                    )
                    postCleanup()
                }
            }
        }
        aborted {
            node(AGENT_LABEL) {
                script {
                    def stackNames = [
                    "test-single-${BUILD_ID}-${ARCHITECTURE}-${BUILD_NUMBER}",
                    "test-single-security-${BUILD_ID}-${ARCHITECTURE}-${BUILD_NUMBER}"
                    ]
                    withCredentials([string(credentialsId: 'perf-test-account-id', variable: 'PERF_TEST_ACCOUNT_ID')]) {
                        withAWS(role: 'cfn-set-up', roleAccount: "${PERF_TEST_ACCOUNT_ID}", duration: 900, roleSessionName: 'jenkins-session', region: 'eu-west-1') {
                            try {
                                for (String stackName : stackNames) {
                                    def stack = null
                                    try {
                                        stack = cfnDescribe(stack: stackName)
                                    } catch (Exception) {
                                        echo "Stack '${stackName}' does not exist, nothing to remove"
                                    }
                                    if (stack != null) {
                                        echo "Deleting stack '${stackName}'"
                                        cfnDelete(stack: stackName, pollInterval:1000)
                                    }
                                }
                            } catch (Exception e) {
                                error "Exception occurred while deleting the CloudFormation stack: ${e.toString()}"
                            }
                        }
                    }
                }
                postCleanup()
            }
        }
    }
}