/* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ import java.util.concurrent.Callable import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { distribution = 'oss-zip' opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") opensearch_plugin_version = System.getProperty("bwc.version", "2.8.0.0") buildVersionQualifier = System.getProperty("build.version_qualifier", "") // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" } if (isSnapshot) { opensearch_build += "-SNAPSHOT" } common_utils_version = System.getProperty("common_utils.version", opensearch_build) } repositories { mavenLocal() mavenCentral() maven { url "https://plugins.gradle.org/m2/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } dependencies { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" classpath "org.jacoco:org.jacoco.agent:0.8.5" } } //****************************************************************************/ // Build configurations //****************************************************************************/ plugins { id 'nebula.ospackage' version "8.3.0" id 'checkstyle' } repositories { mavenLocal() mavenCentral() maven { url "https://plugins.gradle.org/m2/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } apply plugin: 'java' apply plugin: 'jacoco' apply plugin: 'idea' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' checkstyle { toolVersion = '8.29' configFile file("checkstyle/checkstyle.xml") } def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster') def usingMultiNode = project.properties.containsKey('numNodes') // Only apply jacoco test coverage if we are running a local single node cluster if (!usingRemoteCluster) { if (!usingMultiNode) { apply from: 'build-tools/plugin-coverage.gradle' } } ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') } sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 allprojects { group 'org.opensearch' version = opensearch_build } publishing { repositories { maven { name = "Snapshots" url = "https://aws.oss.sonatype.org/content/repositories/snapshots" credentials { username "$System.env.SONATYPE_USERNAME" password "$System.env.SONATYPE_PASSWORD" } } } publications { pluginZip(MavenPublication) { publication -> pom { name = "opensearch-asynchronous-search" description = "OpenSearch Asynchronous Search plugin" groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" url = "http://www.apache.org/licenses/LICENSE-2.0.txt" } } developers { developer { name = "OpenSearch" url = "https://github.com/opensearch-project/asynchronous-search" } } } } } } dependencies { testImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" testImplementation "org.opensearch.plugin:lang-painless:${opensearch_version}" testImplementation "org.opensearch.test:framework:${opensearch_version}" compileOnly "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" compileOnly "org.opensearch:opensearch:${opensearch_version}" implementation "org.opensearch:common-utils:${common_utils_version}" configurations.all { resolutionStrategy { force "com.puppycrawl.tools:checkstyle:${project.checkstyle.toolVersion}" } } } compileTestJava { classpath = classpath.filter{ File file -> !file.name.equals( "hamcrest-core-1.3.jar" ) } } check.dependsOn jacocoTestReport opensearchplugin { name 'opensearch-asynchronous-search' description 'Provides support for asynchronous search' classname 'org.opensearch.search.asynchronous.plugin.AsynchronousSearchPlugin' } tasks.named("integTest").configure { it.dependsOn(project.tasks.named("bundlePlugin")) } licenseHeaders.enabled = true dependencyLicenses.enabled = false thirdPartyAudit.enabled = false validateNebulaPom.enabled = false loggerUsageCheck.enabled = false def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() def securityEnabled = System.getProperty("security", "false") == "true" test { systemProperty 'tests.security.manager', 'false' systemProperty 'es.set.netty.runtime.available.processors', 'false' } File repo = file("$buildDir/testclusters/repo") def _numNodes = findProperty('numNodes') as Integer ?: 1 Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); testClusters.integTest { testDistribution = "ARCHIVE" plugin(project.tasks.bundlePlugin.archiveFile) // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug opensearch JVM // since we also support multi node integration tests we increase debugPort per node if (System.getProperty("opensearch.debug") != null) { def debugPort = 5005 nodes.forEach { node -> node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") debugPort += 1 } } } integTest { systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath systemProperty 'buildDir', buildDir.path systemProperty "https", System.getProperty("https", securityEnabled.toString()) systemProperty "user", System.getProperty("user", "admin") systemProperty "password", System.getProperty("password", "admin") // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. doFirst { systemProperty 'cluster.debug', getDebug() // Set number of nodes system property to be used in tests systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' } if (System.getProperty("tests.rest.bwcsuite") == null) { filter { excludeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } } } task integTestRemote(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") if (System.getProperty("tests.rest.bwcsuite") == null) { filter { excludeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } } // Only rest case can run with remote cluster if (System.getProperty("tests.rest.cluster") != null) { filter { includeTestsMatching "org.opensearch.search.asynchronous.rest*" } } } ext.getPluginResource = { download_to_folder, download_from_src -> project.mkdir download_to_folder ant.get(src: download_from_src, dest: download_to_folder, httpusecaches: false) return fileTree(download_to_folder).getSingleFile() } String baseName = "asynSearchCluster" String bwcVersionShort = "2.8.0" String bwcVersion = bwcVersionShort + ".0" String bwcFilePath = "src/test/resources/org/opensearch/search/asynchronous/bwc/" String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/"+ bwcVersionShort + "/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-asynchronous-search-"+ bwcVersion +".zip" getPluginResource(bwcFilePath + bwcVersion, bwcRemoteFile) // Creates two test clusters of previous version and loads opensearch plugin of bwcVersion 2.times { i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" versions = ["2.8.0",opensearch_version] numberOfNodes = 3 plugin(provider(new Callable() { @Override RegularFile call() throws Exception { return new RegularFile() { @Override File getAsFile() { return fileTree(bwcFilePath + opensearch_plugin_version).getSingleFile() } } } })) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' systemProperty "java.library.path", "$rootDir/src/test/resources/org/opensearch/search/asynchronous/lib:$rootDir/jni/release" } } } // upgradeNodeAndPluginToNextVersion(plugins) upgrades plugin on the upgraded node with project.version binary file in bwcFilePath // upgradeAllNodesAndPluginsToNextVersion(plugins) upgrades plugins on all the 3 nodes after upgrading the nodes List> plugins = [ provider(new Callable(){ @Override RegularFile call() throws Exception { return new RegularFile() { @Override File getAsFile() { return fileTree(bwcFilePath + project.version).getSingleFile() } } } }) ] // Ensure the artifact for the current project version is available to be used for the bwc tests task prepareBwcTests { dependsOn bundle doLast { plugins = [ project.getObjects().fileProperty().value(bundle.getArchiveFile()) ] } } // Creates 2 test clusters with 3 nodes of the old version. 2.times { i -> task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { dependsOn 'prepareBwcTests' useCluster testClusters."${baseName}$i" filter { includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' systemProperty 'tests.rest.bwcsuite_round', 'old' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") } } // Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. // This is also used as a one third upgraded cluster for a rolling upgrade. task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { useCluster testClusters."${baseName}0" dependsOn "${baseName}#oldVersionClusterTask0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'first' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") } // Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. // This is used for rolling upgrade. task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#mixedClusterTask" useCluster testClusters."${baseName}0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'second' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") } // Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. // This results in a fully upgraded cluster. // This is used for rolling upgrade. task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#twoThirdsUpgradedClusterTask" useCluster testClusters."${baseName}0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } mustRunAfter "${baseName}#mixedClusterTask" systemProperty 'tests.rest.bwcsuite_cluster', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'third' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") } // Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version // at the same time resulting in a fully upgraded cluster. task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { dependsOn "${baseName}#oldVersionClusterTask1" useCluster testClusters."${baseName}1" doFirst { testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) } filter { includeTestsMatching "org.opensearch.search.asynchronous.bwc.*IT" } systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' systemProperty 'tests.rest.bwcsuite_round', 'fullrestart' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") } // A bwc test suite which runs all the bwc tasks combined. task bwcTestSuite(type: StandaloneRestIntegTestTask) { exclude '**/*Test*' exclude '**/*IT*' dependsOn tasks.named("${baseName}#mixedClusterTask") dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") dependsOn tasks.named("${baseName}#fullRestartClusterTask") } run { useCluster project.testClusters.integTest doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } } apply from: 'build-tools/pkgbuild.gradle' // updateVersion: Task to auto increment to the next development iteration task updateVersion { onlyIf { System.getProperty('newVersion') } doLast { ext.newVersion = System.getProperty('newVersion') println "Setting version to ${newVersion}." // String tokenization to support -SNAPSHOT ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) } }