o d~j@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z ddlmZmZmZdZdZdZeeZejee e fZd/d d Zd d Zd dZddZ d0ddZddZddZ ddZ!GdddZ"GdddZ#GdddZ$Gdd d Z%Gd!d"d"Z&Gd#d$d$Z'Gd%d&d&Z(Gd'd(d(e)Z*Gd)d*d*Z+Gd+d,d,e+Z,Gd-d.d.Z-dS)1N) defaultdict)IncompleteReadErrorReadTimeoutError) SOCKET_ERROR fallocate rename_filei'liPcCsdddt|DS)Ncss|] }ttjVqdSN)randomchoicestring hexdigits).0_rk/private/var/folders/v1/_jykv66s6qd26_69j1njbrl80000gr/T/pip-target-p1gutpg6/lib/python/s3transfer/utils.py .sz(random_file_extension..)joinrange)Z num_digitsrrrrandom_file_extension-srcK*|dvrt|jdr|jdSdSdS)NZ PutObjectZ UploadPartsignal_not_transferring)hasattrbodyrrequestZoperation_namekwargsrrrr1 rcKr)Nrsignal_transferring)rrr rrrrr 8rr cCstt|t|Sr )intmathceilfloat)size part_sizerrrcalculate_num_parts?sr'cCsN||}||dkrd}|durt|d}n||d}d|d|}|S)aCalculate the range parameter for multipart downloads/copies :type part_size: int :param part_size: The size of the part :type part_index: int :param part_index: The index for which this parts starts. This index starts at zero :type num_parts: int :param num_parts: The total number of parts in the transfer :returns: The value to use for Range parameter on downloads or the CopySourceRange parameter for copies r Nzbytes=-)str)r&Z part_index num_parts total_sizeZ start_rangeZ end_rangeZ range_paramrrrcalculate_range_parameterCs   r-cCsDg}|jjjD]}d|}t||r|tjt|||dq|S)a Retrieves callbacks from a subscriber :type transfer_future: s3transfer.futures.TransferFuture :param transfer_future: The transfer future the subscriber is associated to. :type callback_type: str :param callback_type: The type of callback to retrieve from the subscriber. Valid types include: * 'queued' * 'progress' * 'done' :returns: A list of callbacks for the type specified. All callbacks are preinjected with the transfer future. Zon_)future)metaZ call_argsZ subscribersrappend functoolspartialgetattr)Ztransfer_futureZ callback_type callbacks subscriberZ callback_namerrr get_callbacksas  r6cCs |r |D] }||dqdSdS)aCalls all progress callbacks :param callbacks: A list of progress callbacks to invoke :param bytes_transferred: The number of bytes transferred. This is passed to the callbacks. If no bytes were transferred the callbacks will not be invoked because no progress was achieved. It is also possible to receive a negative amount which comes from retrying a transfer request. bytes_transferredNr)r4r8callbackrrrinvoke_progress_callbacks~s   r:cCs*i}|D] \}}||vr|||<q|S)aoGets a dictionary filtered by whitelisted keys :param original_dict: The original dictionary of arguments to source keys and values. :param whitelisted_key: A list of keys to include in the filtered dictionary. :returns: A dictionary containing key/values from the original dictionary whose key was included in the whitelist )items)Z original_dictZwhitelisted_keysZ filtered_dictkeyvaluerrrget_filtered_dicts  r>c@seZdZddZdS)CallArgscKs"|D] \}}t|||qdS)zA class that records call arguments The call arguments must be passed as keyword arguments. It will set each keyword argument as an attribute of the object along with its associated value. N)r;setattr)selfrargr=rrr__init__szCallArgs.__init__N)__name__ __module__ __qualname__rCrrrrr?s r?c@s(eZdZdZddZddZddZdS) FunctionContainerzAn object that contains a function and any args or kwargs to call it When called the provided function will be called with provided args and kwargs. cO||_||_||_dSr _func_args_kwargs)rAfuncargsrrrrrCs zFunctionContainer.__init__cCsd|j|j|jS)Nz'Function: {} with args {} and kwargs {})formatrJrKrLrArrr__repr__s zFunctionContainer.__repr__cCs|j|ji|jSr rIrPrrr__call__szFunctionContainer.__call__N)rDrErF__doc__rCrQrRrrrrrGs  rGc@s<eZdZdZddZeddZddZdd Zd d Z d S) CountCallbackInvokerzAn abstraction to invoke a callback when a shared count reaches zero :param callback: Callback invoke when finalized count reaches zero cCs t|_||_d|_d|_dS)NrF) threadingLock_lock _callback_count _is_finalizedrAr9rrrrCs  zCountCallbackInvoker.__init__cC0|j |jWdS1swYdSr rWrYrPrrr current_counts$z"CountCallbackInvoker.current_countcCsH|j|jr td|jd7_WddS1swYdS)zIncrement the count by onez;Counter has been finalized it can no longer be incremented.r(N)rWrZ RuntimeErrorrYrPrrr increments"zCountCallbackInvoker.incrementcCs|j5|jdkr td|jd8_|jr(|jdkr0|WddSWddSWddS1s;wYdS)zDecrement the count by onerz,Counter is at zero. It cannot dip below zeror(N)rWrYr_rZrXrPrrr decrements  "zCountCallbackInvoker.decrementcCsT|jd|_|jdkr|WddSWddS1s#wYdS)zFinalize the counter Once finalized, the counter never be incremented and the callback can be invoked once the count reaches zero TrN)rWrZrYrXrPrrrfinalizes  "zCountCallbackInvoker.finalizeN) rDrErFrSrCpropertyr^r`rarbrrrrrTs  rTc@s\eZdZdZddZddZ dddZd d Zd d Zd dZ ddZ ddZ ddZ dS)OSUtilscCs tj|Sr )ospathgetsizerAfilenamerrr get_file_size zOSUtils.get_file_sizecCstj||||ddS)NF)enable_callbacks) ReadFileChunk from_filename)rArj start_byter%r4rrropen_file_chunk_readers zOSUtils.open_file_chunk_readerNcCst||||d|dS)NF)r4rmclose_callbacks)rn)rAfileobj chunk_sizefull_file_sizer4rrrrr#open_file_chunk_reader_from_fileobjsz+OSUtils.open_file_chunk_reader_from_fileobjcCs t||Sr )open)rArjmoderrrrw z OSUtils.opencCs&zt|WdStyYdSw)z+Remove a file, noop if file does not exist.N)rfremoveOSErrorrirrr remove_files  zOSUtils.remove_filecCst||dSr )r)rAZcurrent_filenameZ new_filenamerrrrszOSUtils.rename_filecCsXtj|sdSt|j}t|rdSt|rdSt|r#dSt|r*dSdS)aChecks to see if a file is a special UNIX file. It checks if the file is a character special device, block special device, FIFO, or socket. :param filename: Name of the file :returns: True if the file is a special file. False, if is not. FT) rfrgexistsstatst_modeS_ISCHRS_ISBLKS_ISFIFOS_ISSOCK)clsrjrxrrris_special_files      zOSUtils.is_special_filecCsLtjt}tj|}tj|}|d|jt||}tj||Sr ) rfextseprrgdirnamebasename_MAX_FILENAME_LENlenr)rArjsuffixrgnameZ temp_filenamerrrget_temp_filename;s   zOSUtils.get_temp_filenamecCs\z ||d}t||WdWdS1swYWdSty-||w)Nwb)rwrr{r|)rArjr%frrrallocateBs &  zOSUtils.allocater ) rDrErFrrkrqrvrwr|rrrrrrrrrds   rdc@sleZdZddefddZddZeddZdd d Zd d Z dddZ ddZ ddZ ddZ ddZd S)DeferredOpenFilerrbcCs"||_d|_||_||_||_dS)aA class that defers the opening of a file till needed This is useful for deferring opening of a file till it is needed in a separate thread, as there is a limit of how many open files there can be in a single thread for most operating systems. The file gets opened in the following methods: ``read()``, ``seek()``, and ``__enter__()`` :type filename: str :param filename: The name of the file to open :type start_byte: int :param start_byte: The byte to seek to when the file is opened. :type mode: str :param mode: The mode to use to open the file :type open_function: function :param open_function: The function to use to open the file N) _filename_fileobj _start_byte_mode_open_function)rArjrprxZ open_functionrrrrCLs  zDeferredOpenFile.__init__cCs@|jdur||j|j|_|jdkr|j|jdSdSdSNr)rrrrrseekrPrrr_open_if_neededgs  z DeferredOpenFile._open_if_neededcC|jSr )rrPrrrrmszDeferredOpenFile.nameNcCs||j|Sr )rrread)rAamountrrrrqs zDeferredOpenFile.readcCs||j|dSr )rrwrite)rAdatarrrruszDeferredOpenFile.writecCs||j||dSr )rrr)rAwherewhencerrrryszDeferredOpenFile.seekcCs|jdur|jS|jSr )rrtellrPrrrr}s  zDeferredOpenFile.tellcCs|jr |jdSdSr )rcloserPrrrrszDeferredOpenFile.closecCs ||Sr )rrPrrr __enter__szDeferredOpenFile.__enter__cO |dSr rrArNrrrr__exit__rlzDeferredOpenFile.__exit__r r)rDrErFrwrCrrcrrrrrrrrrrrrrKs    rc@seZdZ   d"ddZe  d#ddZddZd$d d Zd d Zd dZ ddZ ddZ d%ddZ ddZ ddZddZddZddZd d!ZdS)&rnNTcCsf||_|j|_|j|j||j|d|_d|_||_|dur"g|_||_||_|dur1||_dSdS)a Given a file object shown below:: |___________________________________________________| 0 | | full_file_size |----chunk_size---| f.tell() :type fileobj: file :param fileobj: File like object :type chunk_size: int :param chunk_size: The max chunk size to read. Trying to read pass the end of the chunk size will behave like you've reached the end of the file. :type full_file_size: int :param full_file_size: The entire content length associated with ``fileobj``. :type callbacks: A list of function(amount_read) :param callbacks: Called whenever data is read from this object in the order provided. :type enable_callbacks: boolean :param enable_callbacks: True if to run callbacks. Otherwise, do not run callbacks :type close_callbacks: A list of function() :param close_callbacks: Called when close is called. The function should take no arguments. )requested_sizerpactual_file_sizerN) rrr_calculate_file_size_size _amount_read _callbacks_callbacks_enabled_close_callbacks)rArsrtrur4rmrrrrrrCs"*  zReadFileChunk.__init__cCs4t|d}||t|j}||||||S)a[Convenience factory function to create from a filename. :type start_byte: int :param start_byte: The first byte from which to start reading. :type chunk_size: int :param chunk_size: The max chunk size to read. Trying to read pass the end of the chunk size will behave like you've reached the end of the file. :type full_file_size: int :param full_file_size: The entire content length associated with ``fileobj``. :type callbacks: function(amount_read) :param callbacks: Called whenever data is read from this object. :type enable_callbacks: bool :param enable_callbacks: Indicate whether to invoke callback during read() calls. :rtype: ``ReadFileChunk`` :return: A new instance of ``ReadFileChunk`` r)rwrrffstatfilenost_size)rrjrprtr4rmr file_sizerrrros " zReadFileChunk.from_filenamecCs||}t||Sr )min)rArsrrprZmax_chunk_sizerrrrs z"ReadFileChunk._calculate_file_sizecCslt|j|jd}|dur|}nt||}|j|}|jt|7_|jdur4|jr4t |jt||Sr) maxrrrrrrrrr:)rArZ amount_leftZamount_to_readrrrrrs  zReadFileChunk.readcC&|t|jdr|jdSdS)Nr )enable_callbackrrr rPrrrr  z!ReadFileChunk.signal_transferringcCr)Nr)disable_callbackrrrrPrrrr rz%ReadFileChunk.signal_not_transferringcC d|_dSNTrrPrrrrryzReadFileChunk.enable_callbackcCr)NFrrPrrrrryzReadFileChunk.disable_callbackrcCs|dvr td|d||j7}|dkr||j7}n |dkr$||j7}|jt||j|jdurT|jrTtt ||j|jd}t |j|j}||}t |j|dt||jd|_dS)N)rr(zinvalid whence (z, should be 0, 1 or 2)r(rrr7) ValueErrorrrrrrrrrrr:)rArrZ bounded_whereZbounded_amount_readrrrrrs    zReadFileChunk.seekcCs0|jdur|jr|jD]}|q |jdSr )rrrrr[rrrr-s zReadFileChunk.closecCrr )rrPrrrr3szReadFileChunk.tellcCrr )rrPrrr__len__6szReadFileChunk.__len__cCs|Sr rrPrrrr>szReadFileChunk.__enter__cOrr rrrrrrArlzReadFileChunk.__exit__cCstgSr )iterrPrrr__iter__DszReadFileChunk.__iter__)NTNrr r)rDrErFrC classmethodrorrr rrrrrrrrrrrrrrrns, = &    rnc@s"eZdZdZdddZddZdS)StreamReaderProgresszr?rGrTrdrrnr ExceptionrrrrrrrrsT      2XC@%b