3 =\\|@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZddlmZGdddeZdS) )print_functionN) OrderedDict) ClientFactory)CFNYAMLHandler)TaskCatExceptionc@s6eZdZdZdZddddeddfddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZddZdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Zd+d,Zd-d.Zd/d0Zd1d2Zd3d4Zd5d6Zd7d8Zd9d:Zd;d<Z d=d>Z!d?d@Z"dGdAdBZ#e$dCdDZ%e$dEdFZ&dS)H CFNAlchemist NFc Csxtjd|_|jjtjtj|_|jjtjtjd|_|jj |j|jj |jddddg|_ ddd d g|_ dd d d g|_ t|jd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_|j|_d|_d|_d|_d|_d|_d|_ d|_!|j"||j#||j$||j%||j&|||j|j'gkrL|jj(dn |j)||j*||j+| |j,|dS)a Construct an Alchemist object. :param input_path: Directory path to the root of the assets :param target_bucket_name: Target S3 bucket to use as replacement and to upload to :param source_bucket_name: Source S3 bucket to search for replacement :param target_key_prefix: Target S3 key prefix to prepend to all object (including an ending forward slash '/') :param output_directory: Directory to save rewritten assets to :param rewrite_mode: Mode for rewriting like CFNAlchemist.OBJECT_REWRITE_MODE or CFNAlchemist.BASIC_REWRITE_MODE :param verbose: Set to True to log debug messages :param dry_run: Set to True to perform a dry run alchemistz4%(asctime)s - %(name)s - %(levelname)s - %(message)sz .templatez.jsonz.yamlz.ymlz.gitz .gitmodulesz .gitignorez.gitattributesciz.ideaz.vs)loggerNFzaws-quickstartz us-east-1zInvalid rewrite_mode.)-logging getLoggerr setLevelINFO StreamHandlerch Formatter formatter setFormatter addHandler _TEMPLATE_EXT_GIT_EXT_EXCLUDED_DIRSr _boto_clients _auth_mode _aws_profile_aws_access_key_id_aws_secret_access_key_aws_session_token _input_path_target_bucket_name_target_key_prefix_output_directoryOBJECT_REWRITE_MODE _rewrite_mode_excluded_prefixes_verbose_dry_run_prod_bucket_name_prod_key_prefix_default_region _file_listset_input_pathset_prod_bucket_nameset_target_bucket_nameset_target_key_prefixset_output_directoryBASIC_REWRITE_MODEerrorset_rewrite_mode set_verbose set_dry_runset_prod_key_prefix) self input_pathtarget_bucket_nameZsource_bucket_nametarget_key_prefixsource_key_prefixoutput_directoryZ rewrite_modeverbosedry_runr@i/private/var/folders/pf/wv4htv3x0qs2c2mp0dnn0kchsvlck3/T/pip-install-emcbgzcf/taskcat/taskcat/deployer.py__init__sP            zCFNAlchemist.__init__cCs>||_|jj|jrtjntj|jj|jr2tjntjdS)N)r'r rr DEBUGrr)r8r>r@r@rAr5lszCFNAlchemist.set_verbosecCs|jS)N)r')r8r@r@rA get_verboseqszCFNAlchemist.get_verbosecCs ||_dS)N)r()r8r?r@r@rAr6tszCFNAlchemist.set_dry_runcCs|jS)N)r()r8r@r@rA get_dry_runwszCFNAlchemist.get_dry_runcCs ||_dS)N)r )r8r9r@r@rAr-zszCFNAlchemist.set_input_pathcCs|jS)N)r )r8r@r@rAget_input_path}szCFNAlchemist.get_input_pathcCs|dk r|jdd|_dS)N/)stripr*)r8r<r@r@rAr7sz CFNAlchemist.set_prod_key_prefixcCs|jS)N)r*)r8r@r@rAget_prod_key_prefixsz CFNAlchemist.get_prod_key_prefixcCs ||_dS)N)r!)r8r:r@r@rAr/sz#CFNAlchemist.set_target_bucket_namecCs|jS)N)r!)r8r@r@rAget_target_bucket_namesz#CFNAlchemist.get_target_bucket_namecCs$|dk r |jdd|_|jdS)NrG)rHr"_set_excluded_key_prefixes)r8r;r@r@rAr0sz"CFNAlchemist.set_target_key_prefixcCs|jS)N)r")r8r@r@rAget_target_key_prefixsz"CFNAlchemist.get_target_key_prefixcCs ||_dS)N)r#)r8r=r@r@rAr1sz!CFNAlchemist.set_output_directorycCs|jS)N)r#)r8r@r@rAget_output_directorysz!CFNAlchemist.get_output_directorycCs ||_dS)N)r%)r8Z rewrite_typer@r@rAr4szCFNAlchemist.set_rewrite_modecCs|jS)N)r%)r8r@r@rAget_rewrite_modeszCFNAlchemist.get_rewrite_modecCs|dk r||_dS)N)r))r8Zprod_bucket_namer@r@rAr.sz!CFNAlchemist.set_prod_bucket_namecCs|jS)N)r))r8r@r@rAget_prod_bucket_namesz!CFNAlchemist.get_prod_bucket_namecCs ||_dS)N)r+)r8regionr@r@rAset_default_regionszCFNAlchemist.set_default_regioncCs|jS)N)r+)r8r@r@rAget_default_regionszCFNAlchemist.get_default_regioncCs<dj|jdj|jdj|jdj|jdj|jg|_dS)Nz{}doc/z{}pics/z{}media/z {}downloads/z {}installers/)formatr"r&)r8r@r@rArKs     z'CFNAlchemist._set_excluded_key_prefixescCs|jS)N)r&)r8r@r@rA_get_excluded_key_prefixessz'CFNAlchemist._get_excluded_key_prefixesc s~|jdkrtd|jjd|jd}|jd}|j|j}|jj dj |ji}xB|j j dj |jdD](t fd d |jDrl|j<qlW|jj|j|jj d j |j|jr|j}n |j|j}i}xl|D]d}tjj|jr|j r|jn|j|j|jd d jd|tjj|j|j|jd d jdjdd<qW|jj|jtt|jt|j}|jj d|jj |tt|jt|j} |jj d|jj | |jj dj |jx<|jD].|kr|jjdj |j|jjd} t j!t"|dj#j$} |jjdj | ||jjdj | || | kr|jr|jj dj n*|jj dj |j%|jj&|n|jjdj nF|jr|jj dj n*|jj dj |j%|jj&|qWxd|D]\t fdd |jDs|jrV|jj dj n|jj d j |j'qWdS)!aL This function uploads all assets to the target S3 bucket name using the target S3 key prefix for each object. A comparison of checksums is done for all object as well to avoid reuploading files that have not changed (this checksum comparison is only effective on non-multi part uploaded files). Nz target_key_prefix cannot be Noner )credential_setrPZs3z#Gathering remote S3 bucket keys {}*z{})ZPrefixc3s|]}|jkVqdS)N)key).0x)objr@rA sz+CFNAlchemist.upload_only..zGathering local keys {}*z\/\rGz*Keys in remote S3 bucket but not in local:z*Keys in local but not in remote S3 bucket:z!Syncing objects to S3 bucket [{}]zLFile [{0}] exists in S3 bucket [{1}]. Verifying MD5 checksum for difference."rbz#S3 MD5 checksum (etag) [{0}]=>[{1}]z#Local MD5 checksum [{0}]=>[{1}]z[WHAT IF DRY RUN]: UPDATE [{0}]z UPDATE [{0}]z)MD5 checksums are the same. Skipping [{}]z[WHAT IF DRY RUN]: CREATE [{0}]z CREATE [{0}]c3s|]}|kVqdS)Nr@)rWrX)_keyr@rArZsz[WHAT IF DRY RUN]: DELETE [{0}]z DELETE [{0}])(r"rrZ get_sessionrRresourceZBucketr!r inforSZobjectsfilteranyrTrVdebugkeysr,_get_file_listr ospathjoinr#r(replacelstriplistsetZe_tagrHhashlibmd5openread hexdigestZObjectZ upload_filedelete) r8Z boto_sessionZ s3_resourceZ upload_bucketZremote_key_dict file_listZlocal_key_dict current_fileZremote_to_local_diffZlocal_to_remote_diffZs3_hashZ local_hashr@)r`rYrA upload_onlysf      d       zCFNAlchemist.upload_onlycCs|j|j}|jjd|jj||jdk r:tj|j|jjdj|j|jjdj|j |jjdj|j |jjdj|j xF|D]<}|jrt |dkrt jj|jt jj|}qt jj|j|j|jddjd }n|}|j|jkrx|jt|jrxt jj|jd rx|jjd j|t|d dd }|j}|jWdQRX|j}|dd*kr|d+d,kr|jjdd}tjt|d dd td}n(|jjdd}tj t|d dd td}|d-kr2t!|tt"gkr$x|j#D]&}|jj$dj||j%||qWnbt!|t&kr>|j%|nH|j'r\|jj(dj||n*|jj(dj||||k rt)j*|||j'r|jjdj|n|jjdj|tjt jj+|dt|dN} |dkr| j,tj-|dd.d!n&|dkr| j,tj.|d"d#d$d#d#d%WdQRX| jnD|j'rN|jj(d&j|n(|jj(d'j|||k rt)j*||q|jjd j|yt|d dd } | j/} WdQRXx$t0| D]\} } |j1| | | <qW|j'r|jjdj|nP|jjdj|tjt jj+|dt|d}|j2| WdQRX|jWqt3k r|j'rp|jjd(j|n|jj(d)j||j4||Yqt5k rYqt6k r}z |WYdd}~XqXqWdS)/a/ This function searches through all the files and rewrites any references of the production S3 bucket name to the target S3 bucket name. This is done by both things like line-by-line basic rewrites or walking the tree of a JSON or YAML document to find the references. zFiles to be worked on:Nz6Production S3 bucket name that we are looking for [{}]z:Replacement S3 bucket name that we are rewriting with [{}]z5Production S3 key prefix that we are looking for [{}]z9Replacement S3 key prefix that we are rewriting with [{}]r\r[z\/z /templateszOpening file [{}]r)newliner{[}]zDetected JSON. Loading file.JSON)object_pairs_hookzDetected YAML. Loading file.YAMLzWorking on node [{}]zI[WHAT IF DRY RUN]: [{0}] Unsupported {1} structure. Skipping but copying.z6[{0}] Unsupported {1} structure. Skipping but copying.z$[WHAT IF DRY RUN]: Writing file [{}]zWriting file [{}]w,: )indent separatorsTF)rZ allow_unicodeZdefault_flow_styleZexplicit_startZ explicit_endzF[WHAT IF DRY RUN]: [{}] Unsupported file format. Skipping but copying.z3[{}] Unsupported file format. Skipping but copying.zn[WHAT IF DRY RUN]: Ran into a (UnicodeDecodeError) problem trying to read the file [{}]. Skipping but copying.z[Ran into a (UnicodeDecodeError) problem trying to read the file [{}]. Skipping but copying.)rzr{)r|r})r~r)rr)7rgr r rbr#rvalidate_output_dirrSr)r!r*r"lenrhrirjbasenamerkrlr%r2endswithtuplerdirnamerqrrcloserHjsonloadrZordered_safe_loadtypedictrfre_recurse_nodesrmr(warningshutilcopyfilesplitwritedumpsZordered_safe_dump readlines enumerate_string_rewriter writelinesUnicodeDecodeError _copy_filer Exception)r8rurvZ output_filetemplateZtemplate_raw_dataZ FILE_FORMATZ template_dataZnode_keyZupdated_templatefZ file_dataindexlineZ updated_fileer@r@rA rewrite_onlys      $         "    zCFNAlchemist.rewrite_onlycCs|j|jdS)z~ This function performs both a rewrite and upload of files by calling each respective function consecutively. N)rrw)r8r@r@rArewrite_and_uploadszCFNAlchemist.rewrite_and_uploadcCs|jsg}tjj|r"|j|ntjj|rxvtj|D]^\}}}x0|D](}|jt|j sJ|jtjj ||qJWx |j D]}||kr~|j |q~Wq:Wnt d||_|jS)Nz)Directory/File is non-existent. Aborting.)r,rhriisfileappendisdirwalkrrrrjrremover)r8r9r,rootdirsfilesZ _current_file directoryr@r@rArgs     zCFNAlchemist._get_file_listcs|jkrtfddd Drp|jkrR|jjdjjdj|j|jS|jjdjjdSq|jjdjjdj|j|jSn6|j kr|jjdjjdj|j |jSSdS) Nc3s|]}|kVqdS)Nr@)rWrX)current_stringr@rArZsz0CFNAlchemist._string_rewriter..s3:http:https:zRewriting [{}]z z5NOT rewriting [{}] because it's not part of this repo)rrr) r)rdr"r rbrSrstriprkr!r*)r8rr@)rrArs   zCFNAlchemist._string_rewritercCst|ttgkrxv|jD]j}|jjd|jj||jjd|jjt|||jjd|jj|||j||||<qWnLt|tkrxVt|D]J\}}|jjd|jjt||jjd|jj||j|||<qW|St|t kr |j |St|t kr(|jjdnt|t t gkrH|jjdnt|tjtjtjtjgkrt|jjdndt|dkr|jjdnH|jjd|jjd |jjt||jjd |jj|td|jjd |S) NzKey: zType: zValue: z+Not much we can do with booleans. Skipping.z*Not much we can do with numbers. Skipping.z+Not much we can do with datetime. Skipping.z(Not much we can do with nulls. Skipping.zUnsupported type.zFailing Type: zFailing Value: zPARSED!)rrrrfr rerrmrstrrboolintfloatdatetimedatetime timedeltar3r)r8Z current_noderV_indexitemr@r@rArsF              zCFNAlchemist._recurse_nodescCs.tjtjj|d||k r*tj||dS)Nr)rrrhrirrr)r8in_fileout_filer@r@rArszCFNAlchemist._copy_filecCs|dk r$|dko|dks$|jjd|r6d|_||_n(|rX|rXd|_||_||_||_nd|_y8|jjdd|j|j|j|j|j d}|j jd }Wnt k rYnt k r}zyN|jj d |jd |jjdd|j|j|j|j|j d}|j jd }Wnnt k r$YnXt k rz}z:|jjd j|j|jjt|t d j|jWYdd}~XnXWYdd}~XnX|jjd ||jjd|jdS)a This function reads the AWS credentials to ensure that the client has right credentials defined to successfully authenticate against an AWS account. It could be either profile name, access key and secret key or none. :param aws_profile: AWS profile name. :param aws_access_key_id: access key ID secret key. :param aws_secret_access_key: AWS secret access key. NzFCannot use aws_profile with aws_access_key_id or aws_secret_access_keyZprofilerf environmentstsr )rUaws_access_key_idaws_secret_access_keyaws_session_tokenZ profile_namerPZAccountzTrying GovCloud region.z us-gov-west-1z'Credential Error - Please check you {}!zAWS AccountNumber: [%s]zAuthenticated via: [%s])r r3rrrrrrgetrRZget_caller_identityrrrrQrSrerrb)r8 aws_profilerrrZ sts_clientaccountrr@r@rA aws_api_initsZ     4zCFNAlchemist.aws_api_initcCstjddd}|jdtdd|jddtd d|jd td d|jd d tdd|jddtdd|jddtdd|jddddd|jdd}|jddddd|jdd dd!d|jd"d#dd$d|jd%dd&d|jd'd(td)d|jd*d+td,d|jd-d.td/d|jd0d1td/d|jd2dd3d|jd4dd5d|j}|jd6k rx|jd6koj|jd6ksx|j d7|j r|j r|j d8|j s|j r|j d6kr|j d9|jrtj|j |_ |S):z This function creates an argparse parser, parses the arguments, and returns an args object. :return: An object from argparse which contains all the args passed in from the command line. r z0AWS Quick Start rewriter and uploader of assets.)prog descriptionr9z2the input path of assets to rewrite and/or upload.)rhelpz-sbz--source-bucket-namez"source S3 bucket name for rewrite.r:z0target S3 bucket name for rewrite and/or upload.z-tz--target-key-prefixz=target S3 key prefix to use. This is required when uploading.z-spz--source-key-prefixz&source S3 key prefix name for rewrite.z-oz--output-directoryzVcustom output directory path. If no path is specified, will overwrite current file(s).z-bz--basic-rewrite store_truez s