V!\c@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl mZddlmZddlZddlmZmZdZdZd ejfd YZejeZejee jjZd"Z e!Z"d d Z#dZ$dZ%de&fdYZ'de!fdYZ(de!fdYZ)de!fdYZ*de!fdYZ+dej,fdYZ-de!fdYZ.de!fdYZ/d e!fd!YZ0dS(#sAbstractions over S3's upload/download operations. This module provides high level abstractions for efficient uploads/downloads. It handles several things for the user: * Automatically switching to multipart transfers when a file is over a specific size threshold * Uploading/downloading a file in parallel * Throttling based on max bandwidth * Progress callbacks to monitor transfers * Retries. While botocore handles retries for streaming uploads, it is not possible for it to handle retries for streaming downloads. This module handles retries for both cases so you don't need to implement any retry logic yourself. This module has a reasonable set of defaults. It also allows you to configure many aspects of the transfer process including: * Multipart threshold size * Max parallel downloads * Max bandwidth * Socket timeouts * Retry amounts There is no support for s3->s3 multipart copies at this time. .. _ref_s3transfer_usage: Usage ===== The simplest way to use this module is: .. code-block:: python client = boto3.client('s3', 'us-west-2') transfer = S3Transfer(client) # Upload /tmp/myfile to s3://bucket/key transfer.upload_file('/tmp/myfile', 'bucket', 'key') # Download s3://bucket/key to /tmp/myfile transfer.download_file('bucket', 'key', '/tmp/myfile') The ``upload_file`` and ``download_file`` methods also accept ``**kwargs``, which will be forwarded through to the corresponding client operation. Here are a few examples using ``upload_file``:: # Making the object public transfer.upload_file('/tmp/myfile', 'bucket', 'key', extra_args={'ACL': 'public-read'}) # Setting metadata transfer.upload_file('/tmp/myfile', 'bucket', 'key', extra_args={'Metadata': {'a': 'b', 'c': 'd'}}) # Setting content type transfer.upload_file('/tmp/myfile.json', 'bucket', 'key', extra_args={'ContentType': "application/json"}) The ``S3Transfer`` clas also supports progress callbacks so you can provide transfer progress to users. Both the ``upload_file`` and ``download_file`` methods take an optional ``callback`` parameter. Here's an example of how to print a simple progress percentage to the user: .. code-block:: python class ProgressPercentage(object): def __init__(self, filename): self._filename = filename self._size = float(os.path.getsize(filename)) self._seen_so_far = 0 self._lock = threading.Lock() def __call__(self, bytes_amount): # To simplify we'll assume this is hooked up # to a single filename. with self._lock: self._seen_so_far += bytes_amount percentage = (self._seen_so_far / self._size) * 100 sys.stdout.write( " %s %s / %s (%.2f%%)" % (self._filename, self._seen_so_far, self._size, percentage)) sys.stdout.flush() transfer = S3Transfer(boto3.client('s3', 'us-west-2')) # Upload /tmp/myfile to s3://bucket/key and print upload progress. transfer.upload_file('/tmp/myfile', 'bucket', 'key', callback=ProgressPercentage('/tmp/myfile')) You can also provide a TransferConfig object to the S3Transfer object that gives you more fine grained control over the transfer. For example: .. code-block:: python client = boto3.client('s3', 'us-west-2') config = TransferConfig( multipart_threshold=8 * 1024 * 1024, max_concurrency=10, num_download_attempts=10, ) transfer = S3Transfer(client, config) transfer.upload_file('/tmp/foo', 'bucket', 'key') iN(tsix(tReadTimeoutError(tIncompleteReadError(tRetriesExceededErrortS3UploadFailedErrorsAmazon Web Servicess0.2.0t NullHandlercBseZdZRS(cCsdS(N((tselftrecord((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pytemits(t__name__t __module__R(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRsiicCsdjdt|DS(Ntcss!|]}tjtjVqdS(N(trandomtchoicetstringt hexdigits(t.0t_((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pys s(tjointrange(t num_digits((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pytrandom_file_extensionscKs2|dkr.t|jdr.|jjndS(Nt PutObjectt UploadParttdisable_callback(RR(thasattrtbodyR(trequesttoperation_nametkwargs((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pytdisable_upload_callbackss cKs2|dkr.t|jdr.|jjndS(NRRtenable_callback(s PutObjects UploadPart(RRR(RRR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pytenable_upload_callbackss tQueueShutdownErrorcBseZRS((R R (((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR!st ReadFileChunkcBseZd edZed edZdZd dZdZ dZ dZ dZ dZ d Zd Zd Zd ZRS(cCsk||_||_|j|jd|d|d||_|jj|jd|_||_||_dS(s Given a file object shown below: |___________________________________________________| 0 | | full_file_size |----chunk_size---| start_byte :type fileobj: file :param fileobj: File like object :type start_byte: int :param start_byte: The first byte from which to start reading. :type chunk_size: int :param chunk_size: The max chunk size to read. Trying to read pass the end of the chunk size will behave like you've reached the end of the file. :type full_file_size: int :param full_file_size: The entire content length associated with ``fileobj``. :type callback: function(amount_read) :param callback: Called whenever data is read from this object. trequested_sizet start_bytetactual_file_sizeiN(t_fileobjt _start_bytet_calculate_file_sizet_sizetseekt _amount_readt _callbackt_callback_enabled(RtfileobjR$t chunk_sizetfull_file_sizetcallbackR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt__init__s    cCs@t|d}tj|jj}|||||||S(sWConvenience factory function to create from a filename. :type start_byte: int :param start_byte: The first byte from which to start reading. :type chunk_size: int :param chunk_size: The max chunk size to read. Trying to read pass the end of the chunk size will behave like you've reached the end of the file. :type full_file_size: int :param full_file_size: The entire content length associated with ``fileobj``. :type callback: function(amount_read) :param callback: Called whenever data is read from this object. :type enable_callback: bool :param enable_callback: Indicate whether to invoke callback during read() calls. :rtype: ``ReadFileChunk`` :return: A new instance of ``ReadFileChunk`` trb(topentostfstattfilenotst_size(tclstfilenameR$R/R1Rtft file_size((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt from_filenamescCs||}t||S(N(tmin(RR.R#R$R%tmax_chunk_size((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR(s cCs|dkr|j|j}nt|j|j|}|jj|}|jt|7_|jdk r|jr|jt|n|S(N( tNoneR)R+R>R&treadtlenR,R-(Rtamounttamount_to_readtdata((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRAs cCs t|_dS(N(tTrueR-(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRscCs t|_dS(N(tFalseR-(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRscCsS|jj|j||jdk rF|jrF|j||jn||_dS(N(R&R*R'R,R@R-R+(Rtwhere((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR*scCs|jjdS(N(R&tclose(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRIscCs|jS(N(R+(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyttell!scCs|jS(N(R)(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt__len__$scCs|S(N((R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt __enter__,scOs|jdS(N(RI(RtargsR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt__exit__/scCs tgS(N(titer(R((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt__iter__2sN(R R R@RFR2t classmethodR=R(RARRR*RIRJRKRLRNRP(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR"s'         tStreamReaderProgresscBs#eZdZddZdZRS(s<Wrapper for a read only stream that adds progress callbacks.cCs||_||_dS(N(t_streamR,(RtstreamR1((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR2=s cOs>|jj||}|jdk r:|jt|n|S(N(RSRAR,R@RB(RRMRtvalue((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRAAsN(R R t__doc__R@R2RA(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRR;s tOSUtilscBs5eZdZdZdZdZdZRS(cCstjj|S(N(R5tpathtgetsize(RR:((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt get_file_sizeIscCstj||||dtS(NR(R"R=RG(RR:R$tsizeR1((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pytopen_file_chunk_readerLs  cCs t||S(N(R4(RR:tmode((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR4QscCs)ytj|Wntk r$nXdS(s+Remove a file, noop if file does not exist.N(R5tremovetOSError(RR:((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt remove_fileTs cCstjj||dS(N(t s3transfertcompatt rename_file(Rtcurrent_filenamet new_filename((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRc]s(R R RZR\R4R`Rc(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRWHs     tMultipartUploadercBsPeZddddgZejjdZdZdZdZ dZ RS( tSSECustomerKeytSSECustomerAlgorithmtSSECustomerKeyMD5t RequestPayercCs(||_||_||_||_dS(N(t_clientt_configt_ost _executor_cls(Rtclienttconfigtosutilt executor_cls((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR2ks   cCsCi}x6|jD](\}}||jkr|||sR sCRetrying exception caught (%s), retrying request, (attempt %s / %s)R|s$EXITING _download_range for part: %si@(RRltnum_download_attemptsR@RRRRkt get_objectRRRORRRBtsocketttimeoutterrorRRRFR(RRRwR:RRR1RRt max_attemptstlast_exceptiontiRt current_indextchunkR((RRs9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs8         c Cs|jj|d}xtr|jj}|tkrMtjddSy*|\}}|j||j |Wqt k r}tjd|dt|jj qXqWWdQXdS(NtwbsCShutdown sentinel received in IO handler, shutting down IO handler.s!Caught exception in IO thread: %sR|( RmR4RFRtgetRRRR*twriteRR(RR:R;ttasktoffsetRER((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR$s        N( R R RRRR2R@RRRRRR(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs    !tTransferConfigcBs(eZdeddedddZRS(ii iidcCs1||_||_||_||_||_dS(N(tmultipart_thresholdRRRR(RRRRRR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR29s     (R R tMBR2(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR8s t S3TransfercBseZdddddgZddddd d d d d ddddddddddgZdddZdddZdZdddZdZ dZ dZ dZ dZ dZdZRS( t VersionIdRhRgRiRjtACLt CacheControltContentDispositiontContentEncodingtContentLanguaget ContentTypetExpirestGrantFullControlt GrantReadt GrantReadACPt GrantWriteACLtMetadatatServerSideEncryptiont StorageClasst SSEKMSKeyIdcCsO||_|dkr!t}n||_|dkrBt}n||_dS(N(RkR@RRlRWt_osutil(RRoRpRq((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR2fs      cCs|dkri}n|j||j|jjj}|jdtdd|jdt dd|j j ||j j kr|j|||||n|j|||||dS(sUpload a file to an S3 object. Variants have also been injected into S3 client, Bucket and Object. You don't have to use S3Transfer.upload_file() directly. srequest-created.s3t unique_idss3upload-callback-disabless3upload-callback-enableN(R@t_validate_all_known_argstALLOWED_UPLOAD_ARGSRktmetateventstregister_firstRt register_lastR RRZRlRt_multipart_uploadt _put_object(RR:RRwR1RuR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRos    c Cs_|jj}||d|jj|d|)}|jjd|d|d||WdQXdS(NiR1RyRzR(RR\RZRkt put_object(RR:RRwR1RuRR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs    cCs|dkri}n|j||j|j|||}|tjt}y |j||||||Wn:tk rt j d|dt |j j |nX|j j||dS(sDownload an S3 object to a file. Variants have also been injected into S3 client, Bucket and Object. You don't have to use S3Transfer.download_file() directly. s<Exception caught in download_file, removing partial file: %sR|N(R@RtALLOWED_DOWNLOAD_ARGSt _object_sizeR5textsepRt_download_fileRRRRFRR`Rc(RRRwR:RuR1Rt temp_filename((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs     cCsN||jjkr1|j||||||n|j|||||dS(N(RlRt_ranged_downloadt _get_object(RRRwR:RRuR1((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR s cCsCx<|D]4}||krtd|dj|fqqWdS(Ns/Invalid extra_args key '%s', must be one of: %ss, (t ValueErrorR(Rtactualtallowedtkwarg((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs   cCs;t|j|j|j}|j||||||dS(N(RRkRlRR(RRRwR:RRuR1t downloader((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR s c Cs|jj}d}x~t|D]p}y|j|||||SWqtjtjtt fk r} t j d| ||dt | }qqXqWt |dS(NsCRetrying exception caught (%s), retrying request, (attempt %s / %s)R|(RlRR@Rt_do_get_objectRRRRRRRRFR( RRRwR:RuR1RRRR((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyR s     c s|jjd|d||}t|d||jj|d7}x-tfddD]}|j|qbWWdQXdS(NRyRzRRcs jdS(Ni (RA((R(s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRsR (RkRRRRR4ROR( RRRwR:RuR1RR;R((Rs9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs cCs |jjd|d||dS(NRyRzt ContentLength(Rkt head_object(RRRwRu((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRs cCs8t|j|j|j}|j|||||dS(N(RfRkRlRR(RR:RRwR1Rutuploader((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRsN(R R RRR@R2RRRR RR R RRR(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyRFsH          i(1RVR5RRtloggingRRR Rtconcurrent.futuresRtbotocore.compatRt6botocore.vendored.requests.packages.urllib3.exceptionsRtbotocore.exceptionsRts3transfer.compatRats3transfer.exceptionsRRt __author__t __version__tHandlerRt getLoggerR Rt addHandlertmovesRRtobjectRRRR RR!R"RRRWRfRRRRR(((s9/tmp/pip-install-usGedi/s3transfer/s3transfer/__init__.pyt}sD                K l