3 =\@s|ddlmZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl mZddlmZGdd d eZdS) )print_functionN)partial)Pool)PrintMsg)S3UploadFailedError)TaskCatExceptionc@sheZdZdZddgZddgZgZdddZdd d ZdddZ ddZ e ddZ dddZ ddZdS) S3SyncaSyncronizes local project files with S3 based on checksums. Excludes hidden files, unpackaged lambda source and taskcat /ci/ files. Uses the Etag as an md5 which introduces the following limitations * Uses undocumented etag algorithm for multipart uploads * Does not work wil files uploaded in the console that use SSE encryption * see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html for more info Does not support buckets with versioning enabled z.*z*.mdzfunctions/source/.privatecCsP|dkr|jd r|d}||_|j|}|j||}|j|||||ddS)z@Syncronizes local file system with an s3 bucket/prefix /)aclN)endswith s3_client_get_local_file_list_get_s3_file_list_sync)selfrbucketprefixpathr flZs3flrh/private/var/folders/pf/wv4htv3x0qs2c2mp0dnn0kchsvlck3/T/pip-install-emcbgzcf/taskcat/taskcat/s3_sync.py__init__,s  zS3Sync.__init__ic Csg}t|d,}x$|j|}|s"P|jtj|qWWdQRXt|dkr^dj|djSdjdd|D}tj|}dj|jt|S) Nrbz"{}"rcss|]}|jVqdS)N)digest).0mrrr Gsz$S3Sync._hash_file..z"{}-{}") openreadappendhashlibmd5lenformat hexdigestjoin)r file_path chunk_sizeZmd5sfpdataZdigestsZ digests_md5rrr _hash_file8s    zS3Sync._hash_fileTcCsi}tjjtjj|}xtj|D]\}}}tjj||d}d}|dkrPd}xtjD]} |j| rXd}PqXW|s$xf|D]^} d} x tj D]} t j | | rd} PqW| sz|d| } |r|j | } nd} | | g||| <qzWq$W|S)Nr Fz./r T) osrabspath expanduserwalkrelpathrexclude_path_prefixes startswith exclude_filesfnmatchr0)rrZinclude_checksums file_listrootdirsfilesr5Z exclude_pathpfileexclude full_pathZchecksumrrrrKs2       zS3Sync._get_local_file_listc Csi}d}d}x|r|s(|jj||d}n|jj|||d}d|krtx0|dD]$}|dt|d}|d||<qLWd|jkr|d}qd}qW|S) NT)BucketPrefix)rBrCZContinuationTokenZContentsKeyETagZNextContinuationTokenF)rZlist_objects_v2r(keys) rrrZobjectsZ is_paginatedZcontinuation_tokenrespr?r5rrrrms  zS3Sync._get_s3_file_listcCs(d}xtjD]}|j|r d}Pq W|S)NFT)rexclude_remote_path_prefixesr7)rZkeepr@rrr_exclude_remotes   zS3Sync._exclude_remotecszgxZ|jD]N}||jkr|j| rtdjtjtj||||jd||iqWxvfddtdt dDD]T}|j j |d|id} d | jkrx&| d D]} ttj d t | qWtd qWg} x`|D]X} d } | |jkrd } n|| d|| krd } | r|| d}| }| j|||gqWt|}t|j||j |d}|j|| |j|jdS)Nz{}[S3: DELETE ]{} s3://{}/{}rDcsg|]}||dqS)ir)r i)remove_from_s3rr sz S3Sync._sync..riZObjects)rBDeleteZErrorszS3 delete error: %sz*Failed to delete one or more files from S3FTr)rrr )rFrIprintr)rwhite rst_colorr%ranger(rZdelete_objectsERRORstrr ThreadPoolr_s3_upload_filemapcloser+)rZ local_listZs3_listrrr threadsZs3_filedresponseerrorZ upload_to_s3 local_fileuploadZ absolute_paths3_pathpoolfuncr)rLrrs6 $      z S3Sync._syncc Cs|\}}}d}x|dkrtdjtjtj|||y |j||||d|idPWqtk r} zV|d7}ttjd| |dkst| t krdt | krt d t j |d WYdd} ~ XqXqWdS) Nrz{}[S3: -> ]{} s3://{}/{}ZACL)Z ExtraArgsrzS3 upload error: %sz(AccessDenied)zFailed to upload to S3)rOr)rrPrQZ upload_file ExceptionrStyperrTrtimesleep) rpathsrrr Zlocal_filenamerr_retryerrrrVs   zS3Sync._s3_upload_fileN)r i )rk)T)rJ)__name__ __module__ __qualname____doc__r8r6rHrr0rr staticmethodrIrrVrrrrrs   " #r) __future__rr&r9r1rf functoolsrZmultiprocessing.dummyrrUZtaskcat.colored_consolerZboto3.exceptionsrZtaskcat.exceptionsrobjectrrrrr s