/*******************************************************************************
* Copyright 2008-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
* this file except in compliance with the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file.
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
* *****************************************************************************
* __ _ _ ___
* ( )( \/\/ )/ __)
* /__\ \ / \__ \
* (_)(_) \/\/ (___/
*
* AWS SDK for .NET
* API Version: 2006-03-01
*
*/
using System;
using System.IO;
using System.Text;
using Amazon.Util;
using Amazon.Runtime.Internal.Auth;
using System.Globalization;
namespace Amazon.Runtime.Internal.Util
{
///
/// Stream wrapper that double-buffers from a wrapped stream and
/// returns the buffered content as a series of signed 'chunks'
/// for the AWS4 ('Signature V4') protocol.
///
public class ChunkedUploadWrapperStream : WrapperStream
{
public static readonly int DefaultChunkSize = 81920;
private const string CLRF = "\r\n";
private const string CHUNK_STRING_TO_SIGN_PREFIX = "AWS4-HMAC-SHA256-PAYLOAD";
private const string CHUNK_SIGNATURE_HEADER = ";chunk-signature=";
private const int SIGNATURE_LENGTH = 64;
private byte[] _inputBuffer;
private readonly byte[] _outputBuffer;
private int _outputBufferPos = -1;
private int _outputBufferDataLen = -1;
private readonly int _wrappedStreamBufferSize;
private bool _wrappedStreamConsumed;
// if this is set, we've exhausted the input stream and are now sending
// back to the client the final termination chunk, after which all Read
// operations will return 0 bytes.
private bool _outputBufferIsTerminatingChunk;
// The reading strategy used by FillInputBuffer against the wrapped stream.
// We prefer to read direct into our _inputBuffer but this isn't compatible
// with wrapped encryption streams, where we need to read into an interim
// buffer and then copy the encrypted content to _inputBuffer
private enum ReadStrategy
{
ReadDirect,
ReadAndCopy
}
private readonly ReadStrategy _readStrategy = ReadStrategy.ReadDirect;
internal ChunkedUploadWrapperStream(Stream stream, int wrappedStreamBufferSize, AWS4SigningResult headerSigningResult)
: base(stream)
{
HeaderSigningResult = headerSigningResult;
PreviousChunkSignature = headerSigningResult?.Signature;
_wrappedStreamBufferSize = wrappedStreamBufferSize;
_inputBuffer = new byte[DefaultChunkSize];
_outputBuffer = new byte[CalculateChunkHeaderLength(DefaultChunkSize)]; // header+data
#if BCL || NETSTANDARD
// if the wrapped stream implements encryption, switch to a read-and-copy
// strategy for filling the chunk buffer
var encryptionStream = SearchWrappedStream(s =>
{
var encryptUploadPartStream = s as EncryptUploadPartStream;
if (encryptUploadPartStream != null)
return true;
var encryptStream = s as EncryptStream;
return encryptStream != null;
});
if (encryptionStream != null)
_readStrategy = ReadStrategy.ReadAndCopy;
#endif
}
///
/// Reads some or all of the processed chunk to the consumer, constructing
/// and streaming a new chunk if more input data is available.
///
///
///
///
///
public override int Read(byte[] buffer, int offset, int count)
{
// if we've no output and it was the special termination chunk,
// we're done otherwise fill the input buffer with enough data
// for the next chunk (or with whatever is left) and construct
// the chunk in the output buffer ready for streaming
if (_outputBufferPos == -1)
{
if (_wrappedStreamConsumed && _outputBufferIsTerminatingChunk)
return 0;
var bytesRead = FillInputBuffer();
ConstructOutputBufferChunk(bytesRead);
_outputBufferIsTerminatingChunk = (_wrappedStreamConsumed && bytesRead == 0);
}
var outputRemaining = _outputBufferDataLen - _outputBufferPos;
if (outputRemaining < count)
count = outputRemaining;
Buffer.BlockCopy(_outputBuffer, _outputBufferPos, buffer, offset, count);
_outputBufferPos += count;
if (_outputBufferPos >= _outputBufferDataLen)
_outputBufferPos = -1;
return count;
}
///
/// Results of the header-signing portion of the request
///
private AWS4SigningResult HeaderSigningResult { get; set; }
///
/// Computed signature of the chunk prior to the one in-flight, in
/// hex
///
private string PreviousChunkSignature { get; set; }
///
/// Computes the derived signature for a chunk of data of given length in the input buffer,
/// placing a formatted chunk with headers, signature and data into the output buffer
/// ready for streaming back to the consumer.
///
///
private void ConstructOutputBufferChunk(int dataLen)
{
// if the input wasn't sufficient to fill the buffer, size it
// down to make the subseqent hashing/computations easier since
// they don't take any length arguments
if (dataLen > 0 && dataLen < _inputBuffer.Length)
{
var temp = new byte[dataLen];
Buffer.BlockCopy(_inputBuffer, 0, temp, 0, dataLen);
_inputBuffer = temp;
}
var chunkHeader = new StringBuilder();
// variable-length size of the embedded chunk data in hex
chunkHeader.Append(dataLen.ToString("X", CultureInfo.InvariantCulture));
if (HeaderSigningResult != null)
{
const string nonsigExtension = "";
// signature-extension
var chunkStringToSign =
CHUNK_STRING_TO_SIGN_PREFIX + "\n" +
HeaderSigningResult.ISO8601DateTime + "\n" +
HeaderSigningResult.Scope + "\n" +
PreviousChunkSignature + "\n" +
AWSSDKUtils.ToHex(AWS4Signer.ComputeHash(nonsigExtension), true) + "\n" +
(dataLen > 0
? AWSSDKUtils.ToHex(AWS4Signer.ComputeHash(_inputBuffer), true)
: AWS4Signer.EmptyBodySha256);
var chunkSignature = AWSSDKUtils.ToHex(AWS4Signer.SignBlob(HeaderSigningResult.SigningKey, chunkStringToSign), true);
PreviousChunkSignature = chunkSignature;
chunkHeader.Append(nonsigExtension + CHUNK_SIGNATURE_HEADER + chunkSignature);
}
chunkHeader.Append(CLRF);
try
{
var header = Encoding.UTF8.GetBytes(chunkHeader.ToString());
var trailer = Encoding.UTF8.GetBytes(CLRF);
var writePos = 0;
Buffer.BlockCopy(header, 0, _outputBuffer, writePos, header.Length);
writePos += header.Length;
if (dataLen > 0)
{
Buffer.BlockCopy(_inputBuffer, 0, _outputBuffer, writePos, dataLen);
writePos += dataLen;
}
Buffer.BlockCopy(trailer, 0, _outputBuffer, writePos, trailer.Length);
_outputBufferPos = 0;
_outputBufferDataLen = header.Length + dataLen + trailer.Length;
}
catch (Exception e)
{
throw new AmazonClientException("Unable to sign the chunked data. " + e.Message, e);
}
}
///
/// Length override to return the true length of the payload plus the metainfo
/// supplied with each chunk
///
public override long Length
{
get
{
return BaseStream == null ? 0 : ComputeChunkedContentLength(BaseStream.Length);
}
}
public override bool CanSeek
{
get
{
return false;
}
}
///
/// Computes the total size of the data payload, including the chunk metadata.
/// Called externally so as to be able to set the correct Content-Length header
/// value.
///
///
///
public static long ComputeChunkedContentLength(long originalLength)
{
if (originalLength < 0)
throw new ArgumentOutOfRangeException("originalLength", "Expected 0 or greater value for originalLength.");
if (originalLength == 0)
return CalculateChunkHeaderLength(0);
var maxSizeChunks = originalLength / DefaultChunkSize;
var remainingBytes = originalLength % DefaultChunkSize;
return maxSizeChunks * CalculateChunkHeaderLength(DefaultChunkSize)
+ (remainingBytes > 0 ? CalculateChunkHeaderLength(remainingBytes) : 0)
+ CalculateChunkHeaderLength(0);
}
///
/// Computes the size of the header data for each chunk.
///
///
///
private static long CalculateChunkHeaderLength(long chunkDataSize)
{
return chunkDataSize.ToString("X", CultureInfo.InvariantCulture).Length
+ CHUNK_SIGNATURE_HEADER.Length
+ SIGNATURE_LENGTH
+ CLRF.Length
+ chunkDataSize
+ CLRF.Length;
}
///
/// Attempt to read sufficient data for a whole chunk from the wrapped stream,
/// returning the number of bytes successfully read to be processed into a chunk
///
private int FillInputBuffer()
{
if (_wrappedStreamConsumed)
return 0;
var inputBufferPos = 0;
if (_readStrategy == ReadStrategy.ReadDirect)
{
while (inputBufferPos < _inputBuffer.Length && !_wrappedStreamConsumed)
{
// chunk buffer size may not align exactly with underlying buffer size
var chunkBufferRemaining = _inputBuffer.Length - inputBufferPos;
if (chunkBufferRemaining > _wrappedStreamBufferSize)
chunkBufferRemaining = _wrappedStreamBufferSize;
var bytesRead = BaseStream.Read(_inputBuffer, inputBufferPos, chunkBufferRemaining);
if (bytesRead == 0)
_wrappedStreamConsumed = true;
else
inputBufferPos += bytesRead;
}
}
else
{
var readBuffer = new byte[_wrappedStreamBufferSize];
while (inputBufferPos < _inputBuffer.Length && !_wrappedStreamConsumed)
{
var bytesRead = BaseStream.Read(readBuffer, 0, _wrappedStreamBufferSize);
if (bytesRead == 0)
_wrappedStreamConsumed = true;
else
{
Buffer.BlockCopy(readBuffer, 0, _inputBuffer, inputBufferPos, bytesRead);
inputBufferPos += bytesRead;
}
}
}
return inputBufferPos;
}
internal override bool HasLength
{
get
{
return HeaderSigningResult != null;
}
}
}
}