/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. * * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ package org.opensearch.ad; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH; import java.io.IOException; import java.io.InputStreamReader; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import javax.net.ssl.SSLEngine; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.function.Factory; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.junit.After; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.rest.OpenSearchRestTestCase; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; /** * ODFE integration test base class to support both security disabled and enabled ODFE cluster. */ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { protected boolean isHttps() { boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); if (isHttps) { // currently only external cluster is supported for security enabled testing if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { throw new RuntimeException("cluster url should be provided for security enabled testing"); } } return isHttps; } @Override protected String getProtocol() { return isHttps() ? "https" : "http"; } @Override protected Settings restAdminSettings() { return Settings .builder() // disable the warning exception for admin client since it's only used for cleanup. .put("strictDeprecationMode", false) .put("http.port", 9200) .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") .build(); } // Utility fn for deleting indices. Should only be used when not allowed in a regular context // (e.g., deleting system indices) protected static void deleteIndexWithAdminClient(String name) throws IOException { Request request = new Request("DELETE", "/" + name); adminClient().performRequest(request); } // Utility fn for checking if an index exists. Should only be used when not allowed in a regular context // (e.g., checking existence of system indices) protected static boolean indexExistsWithAdminClient(String indexName) throws IOException { Request request = new Request("HEAD", "/" + indexName); Response response = adminClient().performRequest(request); return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } @Override protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { String keystore = settings.get(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); if (Objects.nonNull(keystore)) { URI uri = null; try { uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); } catch (URISyntaxException e) { throw new RuntimeException(e); } Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); return new SecureRestClientBuilder(settings, configPath).build(); } else { configureHttpsClient(builder, settings); builder.setStrictDeprecationMode(strictDeprecationMode); return builder.build(); } } else { configureClient(builder, settings); builder.setStrictDeprecationMode(strictDeprecationMode); return builder.build(); } } @SuppressWarnings("unchecked") @After protected void wipeAllODFEIndices() throws IOException { Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType()); try ( XContentParser parser = xContentType .xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, response.getEntity().getContent() ) ) { XContentParser.Token token = parser.nextToken(); List> parserList = null; if (token == XContentParser.Token.START_ARRAY) { parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); } else { parserList = Collections.singletonList(parser.mapOrdered()); } for (Map index : parserList) { String indexName = (String) index.get("index"); if (indexName != null && !".opendistro_security".equals(indexName)) { adminClient().performRequest(new Request("DELETE", "/" + indexName)); } } } } protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); Header[] defaultHeaders = new Header[headers.size()]; int i = 0; for (Map.Entry entry : headers.entrySet()) { defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); } builder.setDefaultHeaders(defaultHeaders); builder.setHttpClientConfigCallback(httpClientBuilder -> { String userName = Optional .ofNullable(System.getProperty("user")) .orElseThrow(() -> new RuntimeException("user name is missing")); String password = Optional .ofNullable(System.getProperty("password")) .orElseThrow(() -> new RuntimeException("password is missing")); BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); final AuthScope anyScope = new AuthScope(null, -1); credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password.toCharArray())); try { final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder .create() .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 .setTlsDetailsFactory(new Factory() { @Override public TlsDetails create(final SSLEngine sslEngine) { return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); } }) .build(); final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder .create() .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) .setTlsStrategy(tlsStrategy) .build(); return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); } catch (Exception e) { throw new RuntimeException(e); } }); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); final TimeValue socketTimeout = TimeValue .parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback(conf -> { Timeout timeout = Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())); conf.setConnectTimeout(timeout); conf.setResponseTimeout(timeout); return conf; }); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } } /** * wipeAllIndices won't work since it cannot delete security index. Use wipeAllODFEIndices instead. */ @Override protected boolean preserveIndicesUponCompletion() { return true; } protected void waitAllSyncheticDataIngested(int expectedSize, String datasetName, RestClient client) throws Exception { int maxWaitCycles = 3; do { Request request = new Request("POST", String.format(Locale.ROOT, "/%s/_search", datasetName)); request .setJsonEntity( String .format( Locale.ROOT, "{\"query\": {" + " \"match_all\": {}" + " }," + " \"size\": 1," + " \"sort\": [" + " {" + " \"timestamp\": {" + " \"order\": \"desc\"" + " }" + " }" + " ]}" ) ); // Make sure all of the test data has been ingested // Expected response: // "_index":"synthetic","_type":"_doc","_id":"10080","_score":null,"_source":{"timestamp":"2019-11-08T00:00:00Z","Feature1":156.30028000000001,"Feature2":100.211205,"host":"host1"},"sort":[1573171200000]} Response response = client.performRequest(request); JsonObject json = JsonParser .parseReader(new InputStreamReader(response.getEntity().getContent(), Charset.defaultCharset())) .getAsJsonObject(); JsonArray hits = json.getAsJsonObject("hits").getAsJsonArray("hits"); if (hits != null && hits.size() == 1 && expectedSize - 1 == hits.get(0).getAsJsonObject().getAsJsonPrimitive("_id").getAsLong()) { break; } else { request = new Request("POST", String.format(Locale.ROOT, "/%s/_refresh", datasetName)); client.performRequest(request); } Thread.sleep(1_000); } while (maxWaitCycles-- >= 0); } }