/*
* Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Globalization;
using Amazon.Runtime.Internal;
using Amazon.Runtime.Internal.Transform;
using Amazon.Runtime.Internal.Util;
using Amazon.Util;
namespace Amazon.Runtime
{
///
/// A retry policy specifies all aspects of retry behavior. This includes conditions when the request should be retried,
/// checks of retry limit, preparing the request before retry and introducing delay (backoff) before retries.
///
public abstract partial class RetryPolicy
{
///
/// Maximum number of retries to be performed.
/// This does not count the initial request.
///
public int MaxRetries { get; protected set; }
///
/// The logger used to log messages.
///
public ILogger Logger { get; set; }
///
/// Checks if a retry should be performed with the given execution context and exception.
///
/// The execution context which contains both the
/// requests and response context.
/// The exception thrown after issuing the request.
/// Returns true if the request should be retried, else false. The exception is retried if it matches with clockskew error codes.
public bool Retry(IExecutionContext executionContext, Exception exception)
{
// Boolean that denotes retries have not exceeded maxretries and request is rewindable
bool canRetry = !RetryLimitReached(executionContext) && CanRetry(executionContext);
// If canRetry is false, we still want to evaluate the exception if its retryable or not,
// is CSM is enabled. This is necessary to set the IsLastExceptionRetryable property on
// CSM Call Attempt. For S3, with the BucketRegion mismatch exception, an overhead of 100-
// 115 ms was added(because of GetPreSignedUrl and Http HEAD requests).
if (canRetry || executionContext.RequestContext.CSMEnabled)
{
var isClockSkewError = IsClockskew(executionContext, exception);
if (isClockSkewError || RetryForException(executionContext, exception))
{
executionContext.RequestContext.IsLastExceptionRetryable = true;
// If CSM is enabled but canRetry was false, we should not retry the request.
// Return false after successfully evaluating the last exception for retryable.
if (!canRetry)
{
return false;
}
return OnRetry(executionContext, isClockSkewError);
}
}
return false;
}
#region Clock skew correction
private static HashSet clockSkewErrorCodes = new HashSet(StringComparer.OrdinalIgnoreCase)
{
"RequestTimeTooSkewed",
"RequestExpired",
"InvalidSignatureException",
"SignatureDoesNotMatch",
"AuthFailure",
"RequestExpired",
"RequestInTheFuture",
};
private const string clockSkewMessageFormat = "Identified clock skew: local time = {0}, local time with correction = {1}, current clock skew correction = {2}, server time = {3}, service endpoint = {4}.";
private const string clockSkewUpdatedFormat = "Setting clock skew correction: new clock skew correction = {0}, service endpoint = {1}.";
private const string clockSkewMessageParen = "(";
private const string clockSkewMessagePlusSeparator = " + ";
private const string clockSkewMessageMinusSeparator = " - ";
private static TimeSpan clockSkewMaxThreshold = TimeSpan.FromMinutes(5);
///
/// Returns true if the request is in a state where it can be retried, else false.
///
/// The execution context which contains both the
/// requests and response context.
/// Returns true if the request is in a state where it can be retried, else false.
public abstract bool CanRetry(IExecutionContext executionContext);
///
/// Return true if the request should be retried for the given exception.
///
/// The execution context which contains both the
/// requests and response context.
/// The exception thrown by the previous request.
/// Return true if the request should be retried.
public abstract bool RetryForException(IExecutionContext executionContext, Exception exception);
///
/// Checks if the retry limit is reached.
///
/// The execution context which contains both the
/// requests and response context.
/// Return false if the request can be retried, based on number of retries.
public abstract bool RetryLimitReached(IExecutionContext executionContext);
///
/// Waits before retrying a request.
///
/// The execution context which contains both the
/// requests and response context.
public abstract void WaitBeforeRetry(IExecutionContext executionContext);
///
/// Virtual method that gets called on a successful request response.
///
/// The execution context which contains both the
/// requests and response context.
public virtual void NotifySuccess(IExecutionContext executionContext)
{
}
///
/// Virtual method that gets called before a retry request is initiated. The value
/// returned is True by default(retry throttling feature is disabled).
///
/// The execution context which contains both the
/// requests and response context.
public virtual bool OnRetry(IExecutionContext executionContext)
{
return true;
}
///
/// Virtual method that gets called before a retry request is initiated. The value
/// returned is True by default(retry throttling feature is disabled).
///
/// The execution context which contains both the
/// requests and response context.
/// true to bypass any attempt to acquire capacity on a retry
public virtual bool OnRetry(IExecutionContext executionContext, bool bypassAcquireCapacity)
{
return true;
}
private bool IsClockskew(IExecutionContext executionContext, Exception exception)
{
var clientConfig = executionContext.RequestContext.ClientConfig;
var ase = exception as AmazonServiceException;
var isHead =
executionContext.RequestContext.Request != null &&
string.Equals(executionContext.RequestContext.Request.HttpMethod, "HEAD", StringComparison.Ordinal);
var isClockskewErrorCode =
ase != null &&
(ase.ErrorCode == null || clockSkewErrorCodes.Contains(ase.ErrorCode));
if (isHead || isClockskewErrorCode)
{
var endpoint = executionContext.RequestContext.Request.Endpoint.ToString();
var realNow = AWSConfigs.utcNowSource();
var correctedNow = CorrectClockSkew.GetCorrectedUtcNowForEndpoint(endpoint);
DateTime serverTime;
// Try getting server time from the headers
bool serverTimeDetermined = TryParseDateHeader(ase, out serverTime);
// If that fails, try to parse it from the exception message
if (!serverTimeDetermined)
serverTimeDetermined = TryParseExceptionMessage(ase, out serverTime);
if (serverTimeDetermined)
{
// using accurate server time, calculate correction if local time is off
serverTime = serverTime.ToUniversalTime();
var diff = correctedNow - serverTime;
var absDiff = diff.Ticks < 0 ? -diff : diff;
if (absDiff > clockSkewMaxThreshold)
{
var newCorrection = serverTime - realNow;
Logger.InfoFormat(clockSkewMessageFormat,
realNow, correctedNow, clientConfig.ClockOffset, serverTime, endpoint);
// Always set the correction, for informational purposes
CorrectClockSkew.SetClockCorrectionForEndpoint(endpoint, newCorrection);
var shouldRetry = AWSConfigs.CorrectForClockSkew && !AWSConfigs.ManualClockCorrection.HasValue;
// Only retry if clock skew correction is not disabled
if (shouldRetry)
{
// Set clock skew correction
Logger.InfoFormat(clockSkewUpdatedFormat, newCorrection, endpoint);
executionContext.RequestContext.IsSigned = false;
return true;
}
}
}
}
return false;
}
private static bool TryParseDateHeader(AmazonServiceException ase, out DateTime serverTime)
{
var webData = GetWebData(ase);
if (webData != null)
{
// parse server time from "Date" header, if possible
var dateValue = webData.GetHeaderValue(HeaderKeys.DateHeader);
if (!string.IsNullOrEmpty(dateValue))
{
if (DateTime.TryParseExact(
dateValue,
AWSSDKUtils.GMTDateFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal,
out serverTime))
{
return true;
}
}
}
serverTime = DateTime.MinValue;
return false;
}
private static bool TryParseExceptionMessage(AmazonServiceException ase, out DateTime serverTime)
{
if (ase != null && !string.IsNullOrEmpty(ase.Message))
{
var message = ase.Message;
// parse server time from exception message, if possible
var parenIndex = message.IndexOf(clockSkewMessageParen, StringComparison.Ordinal);
if (parenIndex >= 0)
{
parenIndex++;
// Locate " + " or " - " separator that follows the server time string
var separatorIndex = message.IndexOf(clockSkewMessagePlusSeparator, parenIndex, StringComparison.Ordinal);
if (separatorIndex < 0)
separatorIndex = message.IndexOf(clockSkewMessageMinusSeparator, parenIndex, StringComparison.Ordinal);
// Get the server time string and parse it
if (separatorIndex > parenIndex)
{
var timestamp = message.Substring(parenIndex, separatorIndex - parenIndex);
if (DateTime.TryParseExact(
timestamp,
AWSSDKUtils.ISO8601BasicDateTimeFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeUniversal,
out serverTime))
{
return true;
}
}
}
}
serverTime = DateTime.MinValue;
return false;
}
private static IWebResponseData GetWebData(AmazonServiceException ase)
{
if (ase != null)
{
Exception e = ase;
do
{
var here = e as HttpErrorResponseException;
if (here != null)
return here.Response;
e = e.InnerException;
} while (e != null);
}
return null;
}
#endregion
}
}