/* * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ using System; using System.Collections.Generic; using System.Globalization; using Amazon.Runtime.Internal; using Amazon.Runtime.Internal.Transform; using Amazon.Runtime.Internal.Util; using Amazon.Util; namespace Amazon.Runtime { /// /// A retry policy specifies all aspects of retry behavior. This includes conditions when the request should be retried, /// checks of retry limit, preparing the request before retry and introducing delay (backoff) before retries. /// public abstract partial class RetryPolicy { /// /// Maximum number of retries to be performed. /// This does not count the initial request. /// public int MaxRetries { get; protected set; } /// /// The logger used to log messages. /// public ILogger Logger { get; set; } /// /// Checks if a retry should be performed with the given execution context and exception. /// /// The execution context which contains both the /// requests and response context. /// The exception thrown after issuing the request. /// Returns true if the request should be retried, else false. The exception is retried if it matches with clockskew error codes. public bool Retry(IExecutionContext executionContext, Exception exception) { // Boolean that denotes retries have not exceeded maxretries and request is rewindable bool canRetry = !RetryLimitReached(executionContext) && CanRetry(executionContext); // If canRetry is false, we still want to evaluate the exception if its retryable or not, // is CSM is enabled. This is necessary to set the IsLastExceptionRetryable property on // CSM Call Attempt. For S3, with the BucketRegion mismatch exception, an overhead of 100- // 115 ms was added(because of GetPreSignedUrl and Http HEAD requests). if (canRetry || executionContext.RequestContext.CSMEnabled) { var isClockSkewError = IsClockskew(executionContext, exception); if (isClockSkewError || RetryForException(executionContext, exception)) { executionContext.RequestContext.IsLastExceptionRetryable = true; // If CSM is enabled but canRetry was false, we should not retry the request. // Return false after successfully evaluating the last exception for retryable. if (!canRetry) { return false; } return OnRetry(executionContext, isClockSkewError); } } return false; } #region Clock skew correction private static HashSet clockSkewErrorCodes = new HashSet(StringComparer.OrdinalIgnoreCase) { "RequestTimeTooSkewed", "RequestExpired", "InvalidSignatureException", "SignatureDoesNotMatch", "AuthFailure", "RequestExpired", "RequestInTheFuture", }; private const string clockSkewMessageFormat = "Identified clock skew: local time = {0}, local time with correction = {1}, current clock skew correction = {2}, server time = {3}, service endpoint = {4}."; private const string clockSkewUpdatedFormat = "Setting clock skew correction: new clock skew correction = {0}, service endpoint = {1}."; private const string clockSkewMessageParen = "("; private const string clockSkewMessagePlusSeparator = " + "; private const string clockSkewMessageMinusSeparator = " - "; private static TimeSpan clockSkewMaxThreshold = TimeSpan.FromMinutes(5); /// /// Returns true if the request is in a state where it can be retried, else false. /// /// The execution context which contains both the /// requests and response context. /// Returns true if the request is in a state where it can be retried, else false. public abstract bool CanRetry(IExecutionContext executionContext); /// /// Return true if the request should be retried for the given exception. /// /// The execution context which contains both the /// requests and response context. /// The exception thrown by the previous request. /// Return true if the request should be retried. public abstract bool RetryForException(IExecutionContext executionContext, Exception exception); /// /// Checks if the retry limit is reached. /// /// The execution context which contains both the /// requests and response context. /// Return false if the request can be retried, based on number of retries. public abstract bool RetryLimitReached(IExecutionContext executionContext); /// /// Waits before retrying a request. /// /// The execution context which contains both the /// requests and response context. public abstract void WaitBeforeRetry(IExecutionContext executionContext); /// /// Virtual method that gets called on a successful request response. /// /// The execution context which contains both the /// requests and response context. public virtual void NotifySuccess(IExecutionContext executionContext) { } /// /// Virtual method that gets called before a retry request is initiated. The value /// returned is True by default(retry throttling feature is disabled). /// /// The execution context which contains both the /// requests and response context. public virtual bool OnRetry(IExecutionContext executionContext) { return true; } /// /// Virtual method that gets called before a retry request is initiated. The value /// returned is True by default(retry throttling feature is disabled). /// /// The execution context which contains both the /// requests and response context. /// true to bypass any attempt to acquire capacity on a retry public virtual bool OnRetry(IExecutionContext executionContext, bool bypassAcquireCapacity) { return true; } private bool IsClockskew(IExecutionContext executionContext, Exception exception) { var clientConfig = executionContext.RequestContext.ClientConfig; var ase = exception as AmazonServiceException; var isHead = executionContext.RequestContext.Request != null && string.Equals(executionContext.RequestContext.Request.HttpMethod, "HEAD", StringComparison.Ordinal); var isClockskewErrorCode = ase != null && (ase.ErrorCode == null || clockSkewErrorCodes.Contains(ase.ErrorCode)); if (isHead || isClockskewErrorCode) { var endpoint = executionContext.RequestContext.Request.Endpoint.ToString(); var realNow = AWSConfigs.utcNowSource(); var correctedNow = CorrectClockSkew.GetCorrectedUtcNowForEndpoint(endpoint); DateTime serverTime; // Try getting server time from the headers bool serverTimeDetermined = TryParseDateHeader(ase, out serverTime); // If that fails, try to parse it from the exception message if (!serverTimeDetermined) serverTimeDetermined = TryParseExceptionMessage(ase, out serverTime); if (serverTimeDetermined) { // using accurate server time, calculate correction if local time is off serverTime = serverTime.ToUniversalTime(); var diff = correctedNow - serverTime; var absDiff = diff.Ticks < 0 ? -diff : diff; if (absDiff > clockSkewMaxThreshold) { var newCorrection = serverTime - realNow; Logger.InfoFormat(clockSkewMessageFormat, realNow, correctedNow, clientConfig.ClockOffset, serverTime, endpoint); // Always set the correction, for informational purposes CorrectClockSkew.SetClockCorrectionForEndpoint(endpoint, newCorrection); var shouldRetry = AWSConfigs.CorrectForClockSkew && !AWSConfigs.ManualClockCorrection.HasValue; // Only retry if clock skew correction is not disabled if (shouldRetry) { // Set clock skew correction Logger.InfoFormat(clockSkewUpdatedFormat, newCorrection, endpoint); executionContext.RequestContext.IsSigned = false; return true; } } } } return false; } private static bool TryParseDateHeader(AmazonServiceException ase, out DateTime serverTime) { var webData = GetWebData(ase); if (webData != null) { // parse server time from "Date" header, if possible var dateValue = webData.GetHeaderValue(HeaderKeys.DateHeader); if (!string.IsNullOrEmpty(dateValue)) { if (DateTime.TryParseExact( dateValue, AWSSDKUtils.GMTDateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out serverTime)) { return true; } } } serverTime = DateTime.MinValue; return false; } private static bool TryParseExceptionMessage(AmazonServiceException ase, out DateTime serverTime) { if (ase != null && !string.IsNullOrEmpty(ase.Message)) { var message = ase.Message; // parse server time from exception message, if possible var parenIndex = message.IndexOf(clockSkewMessageParen, StringComparison.Ordinal); if (parenIndex >= 0) { parenIndex++; // Locate " + " or " - " separator that follows the server time string var separatorIndex = message.IndexOf(clockSkewMessagePlusSeparator, parenIndex, StringComparison.Ordinal); if (separatorIndex < 0) separatorIndex = message.IndexOf(clockSkewMessageMinusSeparator, parenIndex, StringComparison.Ordinal); // Get the server time string and parse it if (separatorIndex > parenIndex) { var timestamp = message.Substring(parenIndex, separatorIndex - parenIndex); if (DateTime.TryParseExact( timestamp, AWSSDKUtils.ISO8601BasicDateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out serverTime)) { return true; } } } } serverTime = DateTime.MinValue; return false; } private static IWebResponseData GetWebData(AmazonServiceException ase) { if (ase != null) { Exception e = ase; do { var here = e as HttpErrorResponseException; if (here != null) return here.Response; e = e.InnerException; } while (e != null); } return null; } #endregion } }