ó V!¶\c@s¿ddlmZddlmZddlZddlZddlZddlZddlm Z ddlm Z ddl m Z m Z ddlmZddlmZejeƒZd efd „ƒYZd efd „ƒYZd efd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZdefd„ƒYZeddgƒZedƒZ ed ƒZ!dS(!iÿÿÿÿ(tfutures(t namedtupleN(tMAXINT(tsix(tCancelledErrortTransferNotDoneError(tFunctionContainer(t TaskSemaphoretBaseTransferFuturecBs2eZed„ƒZd„Zd„Zd„ZRS(cCstdƒ‚dS(s-The metadata associated to the TransferFuturetmetaN(tNotImplementedError(tself((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR scCstdƒ‚dS(smDetermines if a TransferFuture has completed :returns: True if completed. False, otherwise. sdone()N(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytdone$scCstdƒ‚dS(sìWaits until TransferFuture is done and returns the result If the TransferFuture succeeded, it will return the result. If the TransferFuture failed, it will raise the exception associated to the failure. sresult()N(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytresult+scCstdƒ‚dS(s6Cancels the request associated with the TransferFuturescancel()N(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytcancel4s(t__name__t __module__tpropertyR R R R(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRs  tBaseTransferMetacBs5eZed„ƒZed„ƒZed„ƒZRS(cCstdƒ‚dS(s*The call args used in the transfer requestt call_argsN(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR:scCstdƒ‚dS(sThe unique id of the transfert transfer_idN(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR?scCstdƒ‚dS(s.A dictionary that requesters can store data int user_contextN(R (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRDs(RRRRRR(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR9stTransferFuturecBsJeZddd„Zed„ƒZd„Zd„Zd„Zd„Z RS(cCsL||_|dkr$tƒ|_n||_|dkrHtƒ|_ndS(s‚The future associated to a submitted transfer request :type meta: TransferMeta :param meta: The metadata associated to the request. This object is visible to the requester. :type coordinator: TransferCoordinator :param coordinator: The coordinator associated to the request. This object is not visible to the requester. N(t_metatNonet TransferMetat _coordinatortTransferCoordinator(R R t coordinator((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyt__init__Ks    cCs|jS(N(R(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR ^scCs |jjƒS(N(RR (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR bscCs;y|jjƒSWn#tk r6}|jƒ|‚nXdS(N(RR tKeyboardInterruptR(R te((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR es  cCs|jjƒdS(N(RR(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRoscCs5|jƒstdƒ‚n|jj|dtƒdS(s!Sets the exception on the future.s?set_exception can only be called once the transfer is complete.toverrideN(R RRt set_exceptiontTrue(R t exception((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR!rs  N( RRRRRR R R RR!(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRJs   RcBsbeZdZddd„Zed„ƒZed„ƒZed„ƒZed„ƒZ d„Z RS(s'Holds metadata about the TransferFuturecCs(||_||_d|_i|_dS(N(t _call_argst _transfer_idRt_sizet _user_context(R RR((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR}s   cCs|jS(s*The call args used in the transfer request(R$(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRƒscCs|jS(sThe unique id of the transfer(R%(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRˆscCs|jS(s)The size of the transfer request if known(R&(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytsizescCs|jS(s.A dictionary that requesters can store data in(R'(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR’scCs ||_dS(sÚA method to provide the size of a transfer request By providing this value, the TransferManager will not try to call HeadObject or use the use OS to determine the size of the transfer. N(R&(R R(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytprovide_transfer_size—sN( RRt__doc__RRRRRR(RR)(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR{sRcBs eZdZdd„Zd„Zed„ƒZed„ƒZed„ƒZ ed„ƒZ d„Z e d„Z d „Zd ed „Zd „Zd „Zd„Zdd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„ZRS(s*A helper class for managing TransferFuturecCs‘||_d|_d|_d|_tƒ|_g|_g|_t j ƒ|_ t j ƒ|_ t j ƒ|_t j ƒ|_t j ƒ|_dS(Ns not-started(Rt_statusRt_resultt _exceptiontsett_associated_futurest_failure_cleanupst_done_callbackst threadingtEventt _done_eventtLockt_lockt_associated_futures_lockt_done_callbacks_lockt_failure_cleanups_lock(R R((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR£s       cCsd|jj|jfS(Ns%s(transfer_id=%s)(t __class__RR(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyt__repr__±scCs|jS(N(R-(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR#µscCs$|jtj|jƒSWdQXdS(sÜThe list of futures associated to the inprogress TransferFuture Once the transfer finishes this list becomes empty as the transfer is considered done and there should be no running futures left. N(R7tcopyR/(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytassociated_futures¹s cCs|jS(s;The list of callbacks to call when the TransferFuture fails(R0(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytfailure_cleanupsÆscCs|jS(sßThe status of the TransferFuture The currently supported states are: * not-started - Has yet to start. If in this state, a transfer can be canceled immediately and nothing will happen. * queued - SubmissionTask is about to submit tasks * running - Is inprogress. In-progress as of now means that the SubmissionTask that runs the transfer is being executed. So there is no guarantee any transfer requests had been made to S3 if this state is reached. * cancelled - Was cancelled * failed - An exception other than CancelledError was thrown * success - No exceptions were thrown and is done. (R+(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytstatusËscCs/|j d|_||_d|_WdQXdS(s¾Set a result for the TransferFuture Implies that the TransferFuture succeeded. This will always set a result because it is invoked on the final task where there is only ever one final task and it is ran at the very end of a transfer process. So if a result is being set for this final task, the transfer succeeded even if something came a long and canceled the transfer on the final task. tsuccessN(R6RR-R,R+(R R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyt set_resultÝs   cCs<|j-|jƒ s|r2||_d|_nWdQXdS(såSet an exception for the TransferFuture Implies the TransferFuture failed. :param exception: The exception that cause the transfer to fail. :param override: If True, override any existing state. tfailedN(R6R R-R+(R R#R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR!ìs  cCs,|jjtƒ|jr%|j‚n|jS(sìWaits until TransferFuture is done and returns the result If the TransferFuture succeeded, it will return the result. If the TransferFuture failed, it will raise the exception associated to the failure. (R4twaitRR-R,(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR ùs   tcCs|jp|jƒsut}tjd||ƒ||ƒ|_|jdkrVt}nd|_|ru|jƒqunWdQXdS(s­Cancels the TransferFuture :param msg: The message to attach to the cancellation :param exc_type: The type of exception to set for the cancellation s%s cancel(%s) calleds not-startedt cancelledN( R6R tFalsetloggertdebugR-R+R"t announce_done(R tmsgtexc_typetshould_announce_done((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR s    cCs|jdƒdS(s+Sets the TransferFutrue's status to runningtqueuedN(t_transition_to_non_done_state(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytset_status_to_queuedscCs|jdƒdS(s+Sets the TransferFuture's status to runningtrunningN(RN(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytset_status_to_running!scCsE|j6|jƒr2td|j|fƒ‚n||_WdQXdS(Ns=Unable to transition from done state %s to non-done state %s.(R6R t RuntimeErrorR?R+(R t desired_state((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRN%s   cCs\tjd|||jfƒ|j|d|ƒ}|j|ƒ|jt|j|ƒƒ|S(sßSubmits a task to a provided executor :type executor: s3transfer.futures.BoundedExecutor :param executor: The executor to submit the callable to :type task: s3transfer.tasks.Task :param task: The task to submit to the executor :type tag: s3transfer.futures.TaskTag :param tag: A tag to associate to the submitted task :rtype: concurrent.futures.Future :returns: A future representing the submitted task s;Submitting task %s to executor %s for transfer request: %s.ttag(RGRHRtsubmittadd_associated_futuretadd_done_callbackRtremove_associated_future(R texecutorttaskRTtfuture((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRU-s cCs |jdkS(s¥Determines if a TransferFuture has completed :returns: False if status is equal to 'failed', 'cancelled', or 'success'. True, otherwise RBRER@(sfaileds cancelledssuccess(R?(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR HscCs$|j|jj|ƒWdQXdS(s6Adds a future to be associated with the TransferFutureN(R7R/tadd(R R[((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRVPs cCs$|j|jj|ƒWdQXdS(s4Removes a future's association to the TransferFutureN(R7R/tremove(R R[((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRXUs c Os0|j!|jjt|||ŽƒWdQXdS(s7Add a done callback to be invoked when transfer is doneN(R8R1tappendR(R tfunctiontargstkwargs((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRWZs  c Os0|j!|jjt|||ŽƒWdQXdS(s$Adds a callback to call upon failureN(R9R0R^R(R R_R`Ra((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pytadd_failure_cleanupas  cCs7|jdkr|jƒn|jjƒ|jƒdS(sMAnnounce that future is done running and run associated callbacks This will run any failure cleanups if the transfer failed if not they have not been run, allows the result() to be unblocked, and will run any done callbacks associated to the TransferFuture if they have not already been ran. R@N(R?t_run_failure_cleanupsR4R.t_run_done_callbacks(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRIgs  cCs-|j|j|jƒg|_WdQXdS(N(R8t_run_callbacksR1(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRdts cCs-|j|j|jƒg|_WdQXdS(N(R9ReR>R0(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRc|s cCs"x|D]}|j|ƒqWdS(N(t _run_callback(R t callbackstcallback((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRe„s cCs:y |ƒWn(tk r5tjd|dtƒnXdS(NsException raised in %s.texc_info(t ExceptionRGRHR"(R Rh((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRfˆs  N(RRR*RRR;RR#R=R>R?RARFR!R RRRORQRNRUR RVRXRWRbRIRdRcReRf(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR¡s2                 tBoundedExecutorcBs;eZejZddd„Zded„Zed„Z RS(cCsR||_|dkr!|j}n|d|jƒ|_t|ƒ|_||_dS(sDAn executor implentation that has a maximum queued up tasks The executor will block if the number of tasks that have been submitted and is currently working on is past its maximum. :params max_size: The maximum number of inflight futures. An inflight future means that the task is either queued up or is currently being executed. A size of None or 0 means that the executor will have no bound in terms of the number of inflight futures. :params max_num_threads: The maximum number of threads the executor uses. :type tag_semaphores: dict :params tag_semaphores: A dictionary where the key is the name of the tag and the value is the semaphore to use when limiting the number of tasks the executor is processing at a time. :type executor_cls: BaseExecutor :param underlying_executor_cls: The executor class that get bounded by this executor. If None is provided, the concurrent.futures.ThreadPoolExecutor class is used. t max_workersN(t_max_num_threadsRt EXECUTOR_CLSt _executorRt _semaphoret_tag_semaphores(R tmax_sizetmax_num_threadsttag_semaphorest executor_cls((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR”s    cCsu|j}|r|j|}n|j|j|ƒ}t|j|j|ƒ}t|jj|ƒƒ}|j |ƒ|S(s1Submit a task to complete :type task: s3transfer.tasks.Task :param task: The task to run __call__ on :type tag: s3transfer.futures.TaskTag :param tag: An optional tag to associate to the task. This is used to override which semaphore to use. :type block: boolean :param block: True if to wait till it is possible to submit a task. False, if not to wait and raise an error if not able to submit a task. :returns: The future assocaited to the submitted task ( RpRqtacquireRRtreleasetExecutorFutureRoRURW(R RZRTtblockt semaphoret acquire_tokentrelease_callbackR[((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRU´s  cCs|jj|ƒdS(N(Rotshutdown(R RC((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR}ÙsN( RRRtThreadPoolExecutorRnRRR"RUR}(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRk‘s   %RxcBs,eZd„Zd„Zd„Zd„ZRS(cCs ||_dS(s›A future returned from the executor Currently, it is just a wrapper around a concurrent.futures.Future. However, this can eventually grow to implement the needed functionality of concurrent.futures.Future if we move off of the library and not affect the rest of the codebase. :type future: concurrent.futures.Future :param future: The underlying future N(t_future(R R[((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRÞs cCs |jjƒS(N(RR (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR ëscs#‡fd†}|jj|ƒdS(sAdds a callback to be completed once future is done :parm fn: A callable that takes no arguments. Note that is different than concurrent.futures.Future.add_done_callback that requires a single argument for the future. csˆƒS(N((tfuture_passed_to_callback(tfn(s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyt done_callbackøsN(RRW(R RR‚((Rs8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRWîs cCs |jjƒS(N(RR (R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR üs(RRRR RWR (((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRxÝs  t BaseExecutorcBs/eZdZdd„Zd„Zed„ZRS(sABase Executor class implementation needed to work with s3transfercCsdS(N((R Rl((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRscOstdƒ‚dS(Nssubmit()(R (R RR`Ra((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRUscCstdƒ‚dS(Ns shutdown()(R (R RC((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR}sN(RRR*RRRUR"R}(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRƒs  tNonThreadedExecutorcBs#eZdZd„Zed„ZRS(s@A drop-in replacement non-threaded version of ThreadPoolExecutorcOs}tƒ}y |||Ž}|j|ƒWnMtk rxtjƒd\}}tjd|||ƒ|j||ƒnX|S(Nis0Setting exception for %s to %s with traceback %s(tNonThreadedExecutorFutureRARjtsysRiRGRHtset_exception_info(R RR`RaR[R Rttb((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRUs   cCsdS(N((R RC((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR}s(RRR*RUR"R}(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR„ s R…cBsYeZdZd„Zd„Zd„Zd d„Zd„Zd„Z d„Z d„Z RS( s«The Future returned from NonThreadedExecutor Note that this future is **not** thread-safe as it is being used from the context of a non-threaded environment. cCs1d|_d|_d|_t|_g|_dS(N(RR,R-t _tracebackRFt_doneR1(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR&s     cCs||_|jƒdS(N(R,t _set_done(R R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRA-s cCs ||_||_|jƒdS(N(R-R‰R‹(R R#t traceback((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR‡1s  cCs5|jr.tjt|jƒ|j|jƒn|jS(N(R-RtreraisettypeR‰R,(R ttimeout((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR 6s cCs7t|_x|jD]}|j|ƒqWg|_dS(N(R"RŠR1t_invoke_done_callback(R R‚((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR‹<s cCs ||ƒS(N((R R‚((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRBscCs|jS(N(RŠ(R ((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR EscCs-|jr|j|ƒn|jj|ƒdS(N(RŠRR1R^(R R((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyRWHs N( RRR*RRAR‡RR R‹RR RW(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyR… s       tTaskTagtnametin_memory_uploadtin_memory_download("t concurrentRt collectionsRR<tloggingR†R2ts3transfer.compatRRts3transfer.exceptionsRRts3transfer.utilsRRt getLoggerRRGtobjectRRRRRRkRxRƒR„R…R‘tIN_MEMORY_UPLOAD_TAGtIN_MEMORY_DOWNLOAD_TAG(((s8/tmp/pip-install-usGedi/s3transfer/s3transfer/futures.pyt s0    1&ðL# /