/* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ import java.util.concurrent.Callable import org.opensearch.gradle.test.RestIntegTestTask import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask plugins { id 'java' id 'com.netflix.nebula.ospackage' id "io.freefair.lombok" id 'jacoco' id 'java-library' id 'com.diffplug.spotless' version '6.18.0' id 'checkstyle' } ext { opensearchVersion = "${opensearch_version}" isSnapshot = "true" == System.getProperty("build.snapshot", "true") } apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.testclusters' apply plugin: 'opensearch.pluginzip' ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE') } checkstyle { toolVersion = '10.12.1' } lombok { version = "1.18.28" } opensearchplugin { name 'opensearch-ml' description 'machine learning plugin for opensearch' classname 'org.opensearch.ml.plugin.MachineLearningPlugin' } dependencies { implementation project(':opensearch-ml-common') implementation project(':opensearch-ml-algorithms') implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.opensearch:common-utils:${common_utils_version}" implementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") implementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' implementation group: 'com.google.code.gson', name: 'gson', version: '2.9.1' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' implementation "org.apache.logging.log4j:log4j-slf4j-impl:2.19.0" testImplementation group: 'commons-io', name: 'commons-io', version: '2.11.0' checkstyle "com.puppycrawl.tools:checkstyle:${project.checkstyle.toolVersion}" } publishing { publications { pluginZip(MavenPublication) { publication -> pom { name = opensearchplugin.name description = opensearchplugin.description groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" url = "http://www.apache.org/licenses/LICENSE-2.0.txt" } } developers { developer { name = "OpenSearch" url = "https://github.com/opensearch-project/ml-commons" } } } } } repositories { maven { name = "Snapshots" url = "https://aws.oss.sonatype.org/content/repositories/snapshots" credentials { username "$System.env.SONATYPE_USERNAME" password "$System.env.SONATYPE_PASSWORD" } } } } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } //TODO: check which one should be enabled licenseHeaders.enabled = true testingConventions.enabled = false checkstyleTest.enabled = false forbiddenApis.ignoreFailures = false dependencyLicenses.enabled = false thirdPartyAudit.enabled = false forbiddenApisTest.ignoreFailures = true forbiddenApisMain.ignoreFailures = true validateNebulaPom.enabled = false checkstyleMain.enabled = false loggerUsageCheck.enabled = false def _numNodes = findProperty('numNodes') as Integer ?: 1 test { include '**/*Tests.class' systemProperty 'tests.security.manager', 'false' } def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() task integTest(type: RestIntegTestTask) { description = "Run tests against a cluster" testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath } tasks.named("check").configure { dependsOn(integTest) } integTest { dependsOn "bundlePlugin" systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") // Only rest case can run with remote cluster if (System.getProperty("tests.rest.cluster") != null) { filter { includeTestsMatching "org.opensearch.ml.rest.*IT" } } if (System.getProperty("https") == null || System.getProperty("https") == "false") { filter { excludeTestsMatching "org.opensearch.ml.rest.SecureMLRestIT" } } // The 'doFirst' delays till execution time. doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null systemProperty 'cluster.debug', isDebuggingCluster // Set number of nodes system property to be used in tests systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } } testClusters.integTest { testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM // since we also support multi node integration tests we increase debugPort per node if (System.getProperty("opensearch.debug") != null) { def debugPort = 5005 nodes.forEach { node -> node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") debugPort += 1 } } plugin(project.tasks.bundlePlugin.archiveFile) nodes.each { node -> def plugins = node.plugins def firstPlugin = plugins.get(0) plugins.remove(0) plugins.add(firstPlugin) } } task integTestRemote(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") // Only rest case can run with remote cluster if (System.getProperty("tests.rest.cluster") != null) { filter { includeTestsMatching "org.opensearch.ml.rest.*IT" } } } run { doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } useCluster testClusters.integTest } task release(type: Copy, group: 'build') { dependsOn allprojects*.tasks.build from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile())) into "build/plugins/opensearch-ml" includeEmptyDirs = false } jacocoTestReport { reports { xml.getRequired().set(true) csv.getRequired().set(false) html.getRequired().set(true) } dependsOn test } List jacocoExclusions = [ // TODO: add more unit test to meet the minimal test coverage. 'org.opensearch.ml.constant.CommonValue', 'org.opensearch.ml.plugin.MachineLearningPlugin*', 'org.opensearch.ml.indices.MLIndicesHandler', 'org.opensearch.ml.rest.RestMLPredictionAction', 'org.opensearch.ml.profile.MLModelProfile', 'org.opensearch.ml.profile.MLPredictRequestStats', 'org.opensearch.ml.action.deploy.TransportDeployModelAction', 'org.opensearch.ml.model.MLModelManager', 'org.opensearch.ml.stats.MLClusterLevelStat', 'org.opensearch.ml.stats.MLStatLevel', 'org.opensearch.ml.utils.IndexUtils', 'org.opensearch.ml.cluster.MLCommonsClusterManagerEventListener', 'org.opensearch.ml.cluster.DiscoveryNodeHelper.HotDataNodePredicate', 'org.opensearch.ml.cluster.MLCommonsClusterEventListener', 'org.opensearch.ml.task.MLTaskManager', 'org.opensearch.ml.task.MLTrainingTaskRunner', 'org.opensearch.ml.task.MLPredictTaskRunner', 'org.opensearch.ml.task.MLTaskDispatcher', 'org.opensearch.ml.task.MLTrainAndPredictTaskRunner', 'org.opensearch.ml.task.MLExecuteTaskRunner', 'org.opensearch.ml.action.profile.MLProfileTransportAction', 'org.opensearch.ml.action.models.DeleteModelTransportAction.1', 'org.opensearch.ml.rest.RestMLPredictionAction', 'org.opensearch.ml.breaker.DiskCircuitBreaker', 'org.opensearch.ml.autoredeploy.MLModelAutoReDeployer.SearchRequestBuilderFactory' ] jacocoTestCoverageVerification { violationRules { rule { element = 'CLASS' excludes = jacocoExclusions limit { counter = 'BRANCH' minimum = 0.7 } } rule { element = 'CLASS' excludes = jacocoExclusions limit { counter = 'LINE' value = 'COVEREDRATIO' minimum = 0.7 } } } dependsOn jacocoTestReport } check.dependsOn jacocoTestCoverageVerification configurations.all { resolutionStrategy.force 'org.apache.commons:commons-lang3:3.10' resolutionStrategy.force 'commons-logging:commons-logging:1.2' resolutionStrategy.force 'org.objenesis:objenesis:3.2' resolutionStrategy.force 'net.java.dev.jna:jna:5.11.0' resolutionStrategy.force 'org.apache.commons:commons-text:1.10.0' resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.21.9' } apply plugin: 'com.netflix.nebula.ospackage' // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name afterEvaluate { ospackage { packageName = "${name}" release = isSnapshot ? "0.1" : '1' version = "${project.version}" - "-SNAPSHOT" into '/usr/share/opensearch/plugins' from(zipTree(bundlePlugin.archivePath)) { into opensearchplugin.name } user 'root' permissionGroup 'root' fileMode 0644 dirMode 0755 requires('opensearch', versions.opensearch, EQUAL) packager = 'Amazon' vendor = 'Amazon' os = 'LINUX' prefix '/usr' license 'ASL-2.0' maintainer 'OpenSearch ' url 'https://opensearch.org/downloads.html' summary ''' ML plugin for OpenSearch. Github https://github.com/opensearch-project/ml-commons. '''.stripIndent().replace('\n', ' ').trim() } buildRpm { arch = 'NOARCH' dependsOn 'assemble' } buildDeb { arch = 'all' dependsOn 'assemble' } task buildPackages(type: GradleBuild) { tasks = ['build', 'buildRpm', 'buildDeb'] } } spotless { java { removeUnusedImports() importOrder 'java', 'javax', 'org', 'com' eclipse().configFile rootProject.file('.eclipseformat.xml') } } tasks.withType(licenseHeaders.class) { additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' }