/* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ import org.opensearch.gradle.test.RestIntegTestTask import org.apache.tools.ant.taskdefs.condition.Os buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } // This isn't applying from repositories.gradle so repeating git diff it here repositories { mavenLocal() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } } dependencies { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" } } //****************************************************************************/ // Build configurations //****************************************************************************/ plugins { id 'java-library' id 'java-test-fixtures' id 'idea' id 'jacoco' id "com.diffplug.spotless" version "6.3.0" apply false id 'io.freefair.lombok' version '6.4.3' } apply from: 'gradle/formatting.gradle' apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.repositories' ext { isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] } allprojects { group = 'org.opensearch' version = opensearch_version.tokenize('-')[0] + '.0' if (version_qualifier) { version += "-${version_qualifier}" } if (isSnapshot) { version += "-SNAPSHOT" } apply from: rootProject.file('build-tools/repositories.gradle').absoluteFile plugins.withId('java') { sourceCompatibility = targetCompatibility = "11" } afterEvaluate { project.dependencyLicenses.enabled = false project.thirdPartyAudit.enabled = false project.loggerUsageCheck.enabled = false project.forbiddenApis.ignoreFailures = true project.forbiddenPatterns { setEnabled(false) } project.testingConventions.enabled = false project.validateNebulaPom.enabled = false project.licenseFile = rootProject.file('LICENSE.txt') project.noticeFile = rootProject.file('NOTICE.txt') project.forbiddenApis.ignoreFailures = true } } publishing { repositories { maven { name = "Snapshots" url = "https://aws.oss.sonatype.org/content/repositories/snapshots" credentials { username "$System.env.SONATYPE_USERNAME" password "$System.env.SONATYPE_PASSWORD" } } } publications { pluginZip(MavenPublication) { publication -> pom { name = "opensearch-knn" description = "OpenSearch k-NN plugin" groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" url = "http://www.apache.org/licenses/LICENSE-2.0.txt" } } developers { developer { name = "OpenSearch" url = "https://github.com/opensearch-project/k-NN" } } } } } } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster') def usingMultiNode = project.properties.containsKey('numNodes') // Only apply jacoco test coverage if we are running a local single node cluster def integTestDependOnJniLib = false if (!usingRemoteCluster) { integTestDependOnJniLib = true if (!usingMultiNode) { apply from: 'build-tools/knnplugin-coverage.gradle' } } jacoco { toolVersion = "0.8.7" } check.dependsOn spotlessCheck check.dependsOn jacocoTestReport opensearchplugin { name 'opensearch-knn' // zip file name and plugin name in ${opensearch.plugin.name} read by OpenSearch when plugin loading description 'OpenSearch k-NN plugin' classname 'org.opensearch.knn.plugin.KNNPlugin' extendedPlugins = ['lang-painless'] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') } tasks.named("integTest").configure { it.dependsOn(project.tasks.named("bundlePlugin")) } task release(type: Copy, group: 'build') { dependsOn project.tasks.build from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile())) into "build/plugins/opensearch-knn" includeEmptyDirs = false eachFile { it.path = it.path - "opensearch/" } } //****************************************************************************/ // Dependencies //****************************************************************************/ dependencies { api "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}" api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1' api group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.3' testFixturesImplementation "org.opensearch:common-utils:${version}" } def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() task windowsPatches(type:Exec) { commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1" } task cmakeJniLib(type:Exec) { workingDir 'jni' if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn windowsPatches commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll" } else { commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}" } } task buildJniLib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss' } test { dependsOn buildJniLib systemProperty 'tests.security.manager', 'false' systemProperty "java.library.path", "$rootDir/jni/release" //this change enables mockito-inline that supports mocking of static classes/calls systemProperty "jdk.attach.allowAttachSelf", true if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") } } def _numNodes = findProperty('numNodes') as Integer ?: 1 integTest { if (integTestDependOnJniLib) { dependsOn buildJniLib } systemProperty 'tests.security.manager', 'false' systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath systemProperty "java.library.path", "$rootDir/jni/release" // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null systemProperty 'cluster.debug', isDebuggingCluster // Set number of nodes system property to be used in tests systemProperty 'cluster.number_of_nodes', "${_numNodes}" // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } // The -Ddebug.es option makes the cluster debuggable; this makes the tests debuggable if (System.getProperty("test.debug") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' } } testClusters.integTest { testDistribution = "ARCHIVE" plugin(project.tasks.bundlePlugin.archiveFile) if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") } // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug opensearch JVM // since we also support multi node integration tests we increase debugPort per node if (System.getProperty("cluster.debug") != null) { def debugPort = 5005 nodes.forEach { node -> node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}") debugPort += 1 } } systemProperty("java.library.path", "$rootDir/jni/release") } task integTestRemote(type: RestIntegTestTask) { testClassesDirs = sourceSets.test.output.classesDirs classpath = sourceSets.test.runtimeClasspath systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") systemProperty 'cluster.number_of_nodes', "${_numNodes}" systemProperty 'tests.security.manager', 'false' // Run tests with remote cluster only if rest case is defined if (System.getProperty("tests.rest.cluster") != null) { filter { includeTestsMatching "org.opensearch.knn.*IT" } } } run { useCluster project.testClusters.integTest dependsOn buildJniLib doFirst { // There seems to be an issue when running multi node run or integ tasks with unicast_hosts // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } } } // updateVersion: Task to auto increment to the next development iteration task updateVersion { onlyIf { System.getProperty('newVersion') } doLast { ext.newVersion = System.getProperty('newVersion') println "Setting version to ${newVersion}." // String tokenization to support -SNAPSHOT // Include the required files that needs to be updated with new Version ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { fileset(dir: projectDir) { // Include the required files that needs to be updated with new Version include(name: ".github/workflows/backwards_compatibility_tests_workflow.yml") } } ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) ext.os_version_without_snapshot = opensearch_version.tokenize('-')[0] ext.os_version_major = os_version_without_snapshot.tokenize('.')[0] ext.os_version_minor = os_version_without_snapshot.tokenize('.')[1] ext.os_version_patch = os_version_without_snapshot.tokenize('.')[2] // This condition will check if the BWC workflow is already updated or not and will run next steps if not updated if (!fileTree(".github/workflows/backwards_compatibility_tests_workflow.yml").getSingleFile().text.contains(os_version_without_snapshot)) { // Extract the oldBWCVersion from the existing OpenSearch Version (oldBWCVersion = major . (minor-1) . patch) ext.oldBWCVersion = os_version_major + '.' + Integer.toString(Integer.valueOf(os_version_minor) - 1) + '.' + os_version_patch // Include the current OpenSearch Version before version bump to the bwc_version matrix ant.replaceregexp(file:".github/workflows/backwards_compatibility_tests_workflow.yml", match: oldBWCVersion, replace: oldBWCVersion + '", "' + opensearch_version.tokenize('-')[0], flags:'g', byline:true) } } }