/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.opensearch.hadoop.gradle.buildtools; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.DependencySet; import org.gradle.api.artifacts.ModuleVersionIdentifier; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.internal.ConventionTask; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; /** * A task to gather information about the dependencies and export them into a csv file. *

* The following information is gathered: *

*/ public class DependenciesInfoTask extends ConventionTask { /** * Directory to read license files */ @Optional @InputDirectory private File licensesDir = new File(getProject().getProjectDir(), "licenses").exists() ? new File(getProject().getProjectDir(), "licenses") : null; @OutputFile private File outputFile = new File(getProject().getBuildDir(), "reports/dependencies/dependencies.csv"); private LinkedHashMap mappings; public Configuration getRuntimeConfiguration() { return runtimeConfiguration; } public void setRuntimeConfiguration(Configuration runtimeConfiguration) { this.runtimeConfiguration = runtimeConfiguration; } public Configuration getCompileOnlyConfiguration() { return compileOnlyConfiguration; } public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) { this.compileOnlyConfiguration = compileOnlyConfiguration; } public File getLicensesDir() { return licensesDir; } public void setLicensesDir(File licensesDir) { this.licensesDir = licensesDir; } public File getOutputFile() { return outputFile; } public void setOutputFile(File outputFile) { this.outputFile = outputFile; } /** * Dependencies to gather information from. */ @InputFiles private Configuration runtimeConfiguration; /** * We subtract compile-only dependencies. */ @InputFiles private Configuration compileOnlyConfiguration; public DependenciesInfoTask() { setDescription("Create a CSV file with dependencies information."); } @TaskAction public void generateDependenciesInfo() throws IOException { final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies(); // we have to resolve the transitive dependencies and create a group:artifactId:version map final Set compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration() .getResolvedArtifacts() .stream() .map(r -> { ModuleVersionIdentifier id = r.getModuleVersion().getId(); return id.getGroup() + ":" + id.getName() + ":" + id.getVersion(); }) .collect(Collectors.toSet()); final StringBuilder output = new StringBuilder(); for (final Dependency dep : runtimeDependencies) { // we do not need compile-only dependencies here if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) { continue; } // only external dependencies are checked if (dep instanceof ProjectDependency) { continue; } final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion()); final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName()); getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info"); final String licenseType = getLicenseType(dep.getGroup(), dependencyName); output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n"); } Files.write(outputFile.toPath(), output.toString().getBytes("UTF-8"), StandardOpenOption.CREATE); } @Input public LinkedHashMap getMappings() { return mappings; } public void setMappings(LinkedHashMap mappings) { this.mappings = mappings; } /** * Create an URL on Maven Central * based on dependency coordinates. */ protected String createURL(final String group, final String name, final String version) { final String baseURL = "https://repo1.maven.org/maven2"; return baseURL + "/" + group.replaceAll("\\.", "/") + "/" + name + "/" + version; } /** * Read the LICENSE file associated with the dependency and determine a license type. *

* The license type is one of the following values: *

    *
  • UNKNOWN if LICENSE file is not present for this dependency.
  • *
  • one SPDX identifier if the LICENSE content matches with an SPDX license.
  • *
  • Custom;URL if it's not an SPDX license, * URL is the Github URL to the LICENSE file in opensearch repository.
  • *
* * @param group dependency group * @param name dependency name * @return SPDX identifier, UNKNOWN or a Custom license */ protected String getLicenseType(final String group, final String name) throws IOException { final File license = getDependencyInfoFile(group, name, "LICENSE"); String licenseType; final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license); if (licenseInfo.isSpdxLicense() == false) { // License has not be identified as SPDX. // As we have the license file, we create a Custom entry with the URL to this license file. final String gitBranch = System.getProperty("build.branch", "master"); final String githubBaseURL = "https://raw.githubusercontent.com/elastic/elasticsearch/" + gitBranch + "/"; licenseType = licenseInfo.getIdentifier() + ";" + license.getCanonicalPath().replaceFirst(".*/elasticsearch/", githubBaseURL) + ","; } else { licenseType = licenseInfo.getIdentifier() + ","; } if (licenseInfo.isSourceRedistributionRequired()) { final File sources = getDependencyInfoFile(group, name, "SOURCES"); licenseType += Files.readString(sources.toPath()).trim(); } return licenseType; } protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) { java.util.Optional license = licensesDir != null ? Arrays.stream(licensesDir.listFiles((dir, fileName) -> Pattern.matches(".*-" + infoFileSuffix + ".*", fileName))) .filter(file -> { String prefix = file.getName().split("-" + infoFileSuffix + ".*")[0]; return group.contains(prefix) || name.contains(prefix); }) .findFirst() : java.util.Optional.empty(); return license.orElseThrow( () -> new IllegalStateException( "Unable to find " + infoFileSuffix + " file for dependency " + group + ":" + name + " in " + getLicensesDir().getAbsolutePath() ) ); } }