/* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using OpenSearch.OpenSearch.Xunit;
using Tests.Configuration;
namespace Tests.Core.Xunit
{
/// Feeding TestClient.Configuration options to the runner
public class OpenSearchClientXunitRunOptions : OpenSearchXunitRunOptions
{
public OpenSearchClientXunitRunOptions()
{
RunIntegrationTests = TestConfiguration.Instance.RunIntegrationTests;
RunUnitTests = TestConfiguration.Instance.RunUnitTests;
ClusterFilter = TestConfiguration.Instance.ClusterFilter;
TestFilter = TestConfiguration.Instance.TestFilter;
Version = TestConfiguration.Instance.OpenSearchVersion;
IntegrationTestsMayUseAlreadyRunningNode = TestConfiguration.Instance.TestAgainstAlreadyRunningOpenSearch;
Generators.Initialize();
}
public override void OnBeforeTestsRun() => TestConfiguration.Instance.DumpConfiguration();
public override void OnTestsFinished(Dictionary clusterTotals, ConcurrentBag> failedCollections)
{
DumpClusterTotals(clusterTotals);
DumpSeenDeprecations();
DumpFailedCollections(failedCollections);
}
private static void DumpClusterTotals(Dictionary clusterTotals)
{
Console.WriteLine("--------");
Console.WriteLine("Individual cluster running times:");
foreach (var kv in clusterTotals) Console.WriteLine($"- {kv.Key}: {kv.Value.Elapsed}");
Console.WriteLine("--------");
}
private static void DumpSeenDeprecations()
{
if (XunitRunState.SeenDeprecations.Count == 0) return;
Console.WriteLine("-------- SEEN DEPRECATIONS");
foreach (var d in XunitRunState.SeenDeprecations.Distinct())
Console.WriteLine(d);
Console.WriteLine("--------");
}
private static void DumpFailedCollections(ConcurrentBag> failedCollections)
{
if (failedCollections.Count <= 0)
{
var config = TestConfiguration.Instance;
var runningIntegrations = config.RunIntegrationTests;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("---Reproduce: -----");
var reproduceLine = ReproduceCommandLine(failedCollections, config, runningIntegrations);
Console.WriteLine(reproduceLine);
Console.WriteLine("---------------");
Console.ResetColor();
return;
};
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed collections:");
foreach (var t in failedCollections.OrderBy(p => p.Item1).ThenBy(t => t.Item2))
{
var cluster = t.Item1;
Console.WriteLine($" - {cluster}: {t.Item2}");
}
DumpReproduceFilters(failedCollections);
Console.ResetColor();
}
private static void DumpReproduceFilters(ConcurrentBag> failedCollections)
{
var config = TestConfiguration.Instance;
var runningIntegrations = config.RunIntegrationTests;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("---Reproduce: -----");
var reproduceLine = ReproduceCommandLine(failedCollections, config, runningIntegrations);
Console.WriteLine(reproduceLine);
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION")))
Console.WriteLine($"##teamcity[buildProblem description='{reproduceLine}']");
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD")))
{
var count = failedCollections.Count;
Console.WriteLine($"##vso[task.logissue type=error;]{count} test failures");
Console.WriteLine($"##vso[task.logissue type=error;]{reproduceLine}");
}
Console.WriteLine("--------");
}
private static string ReproduceCommandLine(ConcurrentBag> failedCollections, TestConfigurationBase config,
bool runningIntegrations
)
{
var sb = new StringBuilder(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".\\build.bat " : "./build.sh ")
.Append("seed:").Append(config.Seed).Append(" ");
AppendConfig(nameof(RandomConfiguration.SourceSerializer), config.Random.SourceSerializer, sb);
AppendConfig(nameof(RandomConfiguration.TypedKeys), config.Random.TypedKeys, sb);
AppendConfig(nameof(RandomConfiguration.HttpCompression), config.Random.HttpCompression, sb);
if (runningIntegrations)
sb.Append("integrate ")
.Append(TestConfiguration.Instance.OpenSearchVersion);
else
sb.Append("test");
if (runningIntegrations && failedCollections.Count > 0)
{
var clusters = string.Join(",", failedCollections
.Select(c => c.Item1.ToLowerInvariant())
.Distinct());
sb.Append(" \"");
sb.Append(clusters);
sb.Append("\"");
}
if ((!runningIntegrations || failedCollections.Count < 30) && failedCollections.Count > 0)
{
sb.Append(" \"");
var tests = string.Join(",", failedCollections
.OrderBy(t => t.Item2)
.Select(c => c.Item2.ToLowerInvariant()
.Split('.')
.Last()
.Replace("apitests", "")
.Replace("usagetests", "")
.Replace("tests", "")
));
sb.Append(tests);
sb.Append("\"");
}
var reproduceLine = sb.ToString();
return reproduceLine;
}
///
/// Append random values used
///
private static void AppendConfig(string key, bool value, StringBuilder sb) =>
sb.Append($"random:{key.ToLowerInvariant()}{(value ? "" : ":false")} ");
}
}