/* SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. * * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch B.V. licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ using System; using System.Collections.Concurrent; using System.Diagnostics; using System.Net.Http; using System.Threading; namespace OpenSearch.Net { /// /// Heavily modified version of DefaultHttpClientFactory, re-purposed for RequestData /// https://github.com/dotnet/runtime/blob/master/src/libraries/Microsoft.Extensions.Http/src/DefaultHttpClientFactory.cs /// internal class RequestDataHttpClientFactory : IDisposable { private readonly Func _createHttpClientHandler; private static readonly TimerCallback CleanupCallback = (s) => ((RequestDataHttpClientFactory)s).CleanupTimer_Tick(); private readonly Func> _entryFactory; // Default time of 10s for cleanup seems reasonable. // Quick math: // 10 distinct named clients * expiry time >= 1s = approximate cleanup queue of 100 items // // This seems frequent enough. We also rely on GC occurring to actually trigger disposal. private readonly TimeSpan _defaultCleanupInterval = TimeSpan.FromSeconds(10); // We use a new timer for each regular cleanup cycle, protected with a lock. Note that this scheme // doesn't give us anything to dispose, as the timer is started/stopped as needed. // // There's no need for the factory itself to be disposable. If you stop using it, eventually everything will // get reclaimed. private Timer _cleanupTimer; private readonly object _cleanupTimerLock; private readonly object _cleanupActiveLock; // Collection of 'active' handlers. // // Using lazy for synchronization to ensure that only one instance of HttpMessageHandler is created // for each name. // private readonly ConcurrentDictionary> _activeHandlers; public int InUseHandlers => _activeHandlers.Count; private int _removedHandlers; public int RemovedHandlers => _removedHandlers; // Collection of 'expired' but not yet disposed handlers. // // Used when we're rotating handlers so that we can dispose HttpMessageHandler instances once they // are eligible for garbage collection. // private readonly ConcurrentQueue _expiredHandlers; private readonly TimerCallback _expiryCallback; public RequestDataHttpClientFactory(Func createHttpClientHandler) { _createHttpClientHandler = createHttpClientHandler; // case-sensitive because named options is. _activeHandlers = new ConcurrentDictionary>(); _entryFactory = (key, requestData) => { return new Lazy(() => CreateHandlerEntry(key, requestData), LazyThreadSafetyMode.ExecutionAndPublication); }; _expiredHandlers = new ConcurrentQueue(); _expiryCallback = ExpiryTimer_Tick; _cleanupTimerLock = new object(); _cleanupActiveLock = new object(); } public HttpClient CreateClient(RequestData requestData) { if (requestData == null) throw new ArgumentNullException(nameof(requestData)); var key = HttpConnection.GetClientKey(requestData); var handler = CreateHandler(key, requestData); var client = new HttpClient(handler, disposeHandler: false); client.Timeout = requestData.RequestTimeout; return client; } private HttpMessageHandler CreateHandler(int key, RequestData requestData) { if (requestData == null) throw new ArgumentNullException(nameof(requestData)); #if NETSTANDARD2_1 var entry = _activeHandlers.GetOrAdd(key, (k, r) => _entryFactory(k, r), requestData).Value; #else var entry = _activeHandlers.GetOrAdd(key, (k) => _entryFactory(k, requestData)).Value; #endif StartHandlerEntryTimer(entry); return entry.Handler; } private ActiveHandlerTrackingEntry CreateHandlerEntry(int key, RequestData requestData) { // Wrap the handler so we can ensure the inner handler outlives the outer handler. var handler = new LifetimeTrackingHttpMessageHandler(_createHttpClientHandler(requestData)); // Note that we can't start the timer here. That would introduce a very very subtle race condition // with very short expiry times. We need to wait until we've actually handed out the handler once // to start the timer. // // Otherwise it would be possible that we start the timer here, immediately expire it (very short // timer) and then dispose it without ever creating a client. That would be bad. It's unlikely // this would happen, but we want to be sure. return new ActiveHandlerTrackingEntry(key, handler, requestData.DnsRefreshTimeout); } private void ExpiryTimer_Tick(object state) { var active = (ActiveHandlerTrackingEntry)state; // The timer callback should be the only one removing from the active collection. If we can't find // our entry in the collection, then this is a bug. var removed = _activeHandlers.TryRemove(active.Key, out var found); if (removed) Interlocked.Increment(ref _removedHandlers); Debug.Assert(removed, "Entry not found. We should always be able to remove the entry"); // ReSharper disable once RedundantNameQualifier Debug.Assert(object.ReferenceEquals(active, found.Value), "Different entry found. The entry should not have been replaced"); // At this point the handler is no longer 'active' and will not be handed out to any new clients. // However we haven't dropped our strong reference to the handler, so we can't yet determine if // there are still any other outstanding references (we know there is at least one). // // We use a different state object to track expired handlers. This allows any other thread that acquired // the 'active' entry to use it without safety problems. var expired = new ExpiredHandlerTrackingEntry(active); _expiredHandlers.Enqueue(expired); StartCleanupTimer(); } protected virtual void StartHandlerEntryTimer(ActiveHandlerTrackingEntry entry) => entry.StartExpiryTimer(_expiryCallback); protected virtual void StartCleanupTimer() { lock (_cleanupTimerLock) _cleanupTimer ??= NonCapturingTimer.Create(CleanupCallback, this, _defaultCleanupInterval, Timeout.InfiniteTimeSpan); } protected virtual void StopCleanupTimer() { lock (_cleanupTimerLock) { _cleanupTimer?.Dispose(); _cleanupTimer = null; } } private void CleanupTimer_Tick() { // Stop any pending timers, we'll restart the timer if there's anything left to process after cleanup. // // With the scheme we're using it's possible we could end up with some redundant cleanup operations. // This is expected and fine. // // An alternative would be to take a lock during the whole cleanup process. This isn't ideal because it // would result in threads executing ExpiryTimer_Tick as they would need to block on cleanup to figure out // whether we need to start the timer. StopCleanupTimer(); if (!Monitor.TryEnter(_cleanupActiveLock)) { // We don't want to run a concurrent cleanup cycle. This can happen if the cleanup cycle takes // a long time for some reason. Since we're running user code inside Dispose, it's definitely // possible. // // If we end up in that position, just make sure the timer gets started again. It should be cheap // to run a 'no-op' cleanup. StartCleanupTimer(); return; } try { var initialCount = _expiredHandlers.Count; for (var i = 0; i < initialCount; i++) { // Since we're the only one removing from _expired, TryDequeue must always succeed. _expiredHandlers.TryDequeue(out var entry); Debug.Assert(entry != null, "Entry was null, we should always get an entry back from TryDequeue"); if (entry.CanDispose) { try { entry.InnerHandler.Dispose(); } catch (Exception) { // ignored (ignored in HttpClientFactory too) } } // If the entry is still live, put it back in the queue so we can process it // during the next cleanup cycle. else _expiredHandlers.Enqueue(entry); } } finally { Monitor.Exit(_cleanupActiveLock); } // We didn't totally empty the cleanup queue, try again later. if (_expiredHandlers.Count > 0) StartCleanupTimer(); } public void Dispose() { //try to cleanup nicely CleanupTimer_Tick(); _cleanupTimer?.Dispose(); //CleanupTimer might not cleanup everything because it will only dispose if the WeakReference allows it. // here we forcefully dispose a Client -> ConnectionSettings -> Connection -> RequestDataHttpClientFactory var attempts = 0; do { attempts++; var initialCount = _expiredHandlers.Count; for (var i = 0; i < initialCount; i++) { // Since we're the only one removing from _expired, TryDequeue must always succeed. _expiredHandlers.TryDequeue(out var entry); try { entry?.InnerHandler.Dispose(); } catch (Exception) { // ignored (ignored in HttpClientFactory too) } } } while (attempts < 5 && _expiredHandlers.Count > 0); } } }