/* * This file was generated by the Gradle 'init' task. * * This is a general purpose Gradle build. * Learn more about Gradle by exploring our samples at https://docs.gradle.org/7.5.1/samples * This project uses @Incubating APIs which are subject to change. */ import org.opensearch.gradle.test.RestIntegTestTask import java.util.concurrent.Callable apply plugin: 'java' apply plugin: 'idea' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.pluginzip' apply plugin: 'jacoco' apply plugin: "com.diffplug.spotless" apply plugin: 'io.freefair.lombok' def pluginName = 'opensearch-neural-search' def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem' def projectPath = 'org.opensearch' def pathToPlugin = 'neuralsearch.plugin' def pluginClassName = 'NeuralSearch' tasks.register("preparePluginPathDirs") { mustRunAfter clean doLast { def newPath = pathToPlugin.replace(".", "/") mkdir "src/main/java/org/opensearch/$newPath" mkdir "src/test/java/org/opensearch/$newPath" } } publishing { repositories { maven { name = "Snapshots" url = "https://aws.oss.sonatype.org/content/repositories/snapshots" credentials { username "$System.env.SONATYPE_USERNAME" password "$System.env.SONATYPE_PASSWORD" } } } publications { pluginZip(MavenPublication) { publication -> pom { name = pluginName description = pluginDescription groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" url = "http://www.apache.org/licenses/LICENSE-2.0.txt" } } developers { developer { name = "OpenSearch" url = "https://github.com/opensearch-project/neural-search" } } } } } } opensearchplugin { name pluginName description pluginDescription classname "${projectPath}.${pathToPlugin}.${pluginClassName}" licenseFile rootProject.file('LICENSE') noticeFile rootProject.file('NOTICE') extendedPlugins = ['opensearch-knn'] } dependencyLicenses.enabled = false thirdPartyAudit.enabled = false loggerUsageCheck.enabled = false // No need to validate pom, as we do not upload to maven/sonatype validateNebulaPom.enabled = false buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") isSnapshot = "true" == System.getProperty("build.snapshot", "true") version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' plugin_no_snapshot = opensearch_build if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" plugin_no_snapshot += "-${buildVersionQualifier}" } if (isSnapshot) { opensearch_build += "-SNAPSHOT" } opensearch_group = "org.opensearch" opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") } repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1" classpath "io.freefair.gradle:lombok-plugin:8.0.1" } } ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") } allprojects { group = opensearch_group version = "${opensearch_build}" targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 } repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } configurations { zipArchive } def knnJarDirectory = "$buildDir/dependencies/opensearch-knn" dependencies { api "org.opensearch:opensearch:${opensearch_version}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" compileOnly fileTree(dir: knnJarDirectory, include: '*.jar') api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' // ml-common excluded reflection for runtime so we need to add it by ourselves. // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9 // TODO: Remove following three lines of dependencies if ml-common include them in their jar runtimeOnly group: 'org.reflections', name: 'reflections', version: '0.9.12' runtimeOnly group: 'org.javassist', name: 'javassist', version: '3.29.2-GA' runtimeOnly group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" } // In order to add the jar to the classpath, we need to unzip the // k-NN zip and then copy it into a directory that we specify as a dependency. task extractKnnJar(type: Copy) { mustRunAfter() from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-knn")})) into knnJarDirectory } project.tasks.delombok.dependsOn(extractKnnJar) compileJava { dependsOn extractKnnJar options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() def _numNodes = findProperty('numNodes') as Integer ?: 1 test { include '**/*Tests.class' systemProperty 'tests.security.manager', 'false' } // Setting up Integration Tests task integTest(type: RestIntegTestTask) { description = "Run tests against a cluster" testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath } tasks.named("check").configure { dependsOn(integTest) } integTest { systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null systemProperty 'cluster.debug', isDebuggingCluster // Set number of nodes system property to be used in tests systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' } } testClusters.integTest { testDistribution = "ARCHIVE" // Install K-NN/ml-commons plugins on the integTest cluster nodes configurations.zipArchive.asFileTree.each { plugin(provider(new Callable(){ @Override RegularFile call() throws Exception { return new RegularFile() { @Override File getAsFile() { return it } } } })) } // This installs our neural-search plugin into the testClusters plugin(project.tasks.bundlePlugin.archiveFile) // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug opensearch JVM // since we also support multi node integration tests we increase debugPort per node if (System.getProperty("cluster.debug") != null) { def debugPort = 5005 nodes.forEach { node -> node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}") debugPort += 1 } } // Increase heap size from default of 512mb to 1gb. When heap size is 512mb, our integ tests sporadically fail due // to ml-commons memory circuit breaker exception jvmArgs("-Xms1g", "-Xmx1g") // enable features for testing // hybrid search systemProperty('neural_search_hybrid_search_enabled', 'true') // search pipelines systemProperty('opensearch.experimental.feature.search_pipeline.enabled', 'true') } // Remote Integration Tests task integTestRemote(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") systemProperty 'cluster.number_of_nodes', "${_numNodes}" systemProperty 'tests.security.manager', 'false' // Run tests with remote cluster only if rest case is defined if (System.getProperty("tests.rest.cluster") != null) { filter { includeTestsMatching "org.opensearch.neuralsearch.*IT" } } } run { useCluster testClusters.integTest } spotless { java { removeUnusedImports() importOrder 'java', 'javax', 'org', 'com' eclipse().configFile rootProject.file('formatterConfig.xml') trimTrailingWhitespace() endWithNewline() } } jacocoTestReport { dependsOn integTest, test reports { xml.required = true html.required = true } } check.dependsOn jacocoTestCoverageVerification jacocoTestCoverageVerification.dependsOn jacocoTestReport // updateVersion: Task to auto update version to the next development iteration task updateVersion { onlyIf { System.getProperty('newVersion') } doLast { ext.newVersion = System.getProperty('newVersion') println "Setting version to ${newVersion}." // String tokenization to support -SNAPSHOT ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) } }