B ?@Sa)V@sddlZddlmZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlZddlmZmZmZddlmZmZmZdd lmZmZdd lmZmZdd lmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%ddl&m'Z'm(Z(e)e*Z+GdddZ,ddde'dde(ddfddZ-Gddde.Z/Gddde!Z0Gddde Z1GdddZ2Gddde2Z3Gd d!d!eZ4Gd"d#d#Z5Gd$d%d%Z6Gd&d'd'Z7Gd(d)d)Z8dS)*N)BytesIO)UNSIGNED)Config)urlsplit)NoCredentialsError)S3Client S3RequestTypeS3RequestTlsMode)ClientBootstrapDefaultHostResolverEventLoopGroup)ClientTlsContextTlsContextOptions)AwsCredentialsProviderAwsCredentials)TransferNotDoneError)BaseTransferFutureBaseTransferMeta)CallArgsOSUtils get_callbacks)GBMBc@s$eZdZddZddZddZdS)CRTCredentialProviderAdaptercCs||_d|_t|_dS)N)_botocore_credential_provider_loaded_credentials threadingLock_lock)selfbotocore_credential_providerr!f/private/var/folders/fg/1jzmct0d7d72tjkvm_1nhqc5sw67yj/T/pip-unpacked-wheel-fzz9nylo/s3transfer/crt.py__init__'sz%CRTCredentialProviderAdapter.__init__cCs|}t|j|j|jS)N)_get_credentialsZget_frozen_credentialsrZ access_keyZ secret_keytoken)r credentialsr!r!r"__call__,s z%CRTCredentialProviderAdapter.__call__c Cs@|j0|jdkr0|j}|dkr*t||_|jSQRXdS)N)rrrZload_credentialsr)rZ loaded_credsr!r!r"r$1s  z-CRTCredentialProviderAdapter._get_credentialsN)__name__ __module__ __qualname__r#r'r$r!r!r!r"r&srTc Cst|}t|}t||} d} d} |r,tjntj} |dk rht} |rR| j|dnd| _t | }| } |r~t |}t |} |dt}t| || || | |dS)a :type region: str :param region: The region used for signing :type botocore_credential_provider: Optional[botocore.credentials.CredentialResolver] :param botocore_credential_provider: Provide credentials for CRT to sign the request if not set, the request will not be signed :type num_threads: Optional[int] :param num_threads: Number of worker threads generated. Default is the number of processors in the machine. :type target_throughput: Optional[int] :param target_throughput: Throughput target in Bytes. Default is 0.625 GB/s (which translates to 5 Gb/s). :type part_size: Optional[int] :param part_size: Size, in Bytes, of parts that files will be downloaded or uploaded in. :type use_ssl: boolean :param use_ssl: Whether or not to use SSL. By default, SSL is used. Note that not all services support non-ssl connections. :type verify: Optional[boolean/string] :param verify: Whether or not to verify SSL certificates. By default SSL certificates are verified. You can provide the following values: * False - do not validate SSL certificates. SSL will still be used (unless use_ssl is False), but SSL certificates will not be verified. * path/to/cert/bundle.pem - A filename of the CA cert bundle to use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. N)Z ca_filepathFr,) bootstrapregionZcredential_provider part_sizetls_modetls_connection_optionsZthroughput_target_gbps)r r r r ZENABLEDZDISABLEDrZ&override_default_trust_store_from_pathZ verify_peerr Znew_connection_optionsrrZ new_delegaterr)r.r Z num_threadsZtarget_throughputr/Zuse_sslverifyZevent_loop_groupZ host_resolverr-Zproviderr1r0Ztls_ctx_optionsZclient_tls_optionZcredentails_provider_adapterZ target_gbpsr!r!r"create_s3_crt_client<s:-    r3c@seZdZdddZddZddZddd Zdd d Zd d d Zd!ddZ ddZ ddZ ddZ d"ddZ ddZddZdS)#CRTTransferManagerNcCs@|dkrt|_||_t||j|_g|_td|_d|_ dS)aFA transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client :param crt_s3_client: The CRT s3 client, handling all the HTTP requests and functions under then hood :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer :param crt_request_serializer: Serializer, generates unsigned crt HTTP request. :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. Nr) r_osutil_crt_s3_clientS3ClientArgsCreator_s3_args_creator_future_coordinatorsr Semaphore _semaphore _id_counter)rZ crt_s3_clientcrt_request_serializerosutilr!r!r"r#s  zCRTTransferManager.__init__cCs|S)Nr!)rr!r!r" __enter__szCRTTransferManager.__enter__cGsd}|r d}||dS)NFT) _shutdown)rexc_type exc_valueargscancelr!r!r"__exit__szCRTTransferManager.__exit__cCs6|dkr i}|dkri}t|||||d}|d|S)N)bucketkeyfileobj extra_args subscribersZ get_object)r_submit_transfer)rrGrHrIrJrKcallargsr!r!r"downloads zCRTTransferManager.downloadcCs6|dkr i}|dkri}t|||||d}|d|S)N)rGrHrIrJrKZ put_object)rrL)rrIrGrHrJrKrMr!r!r"uploads zCRTTransferManager.uploadcCs4|dkr i}|dkri}t||||d}|d|S)N)rGrHrJrKZ delete_object)rrL)rrGrHrJrKrMr!r!r"deleteszCRTTransferManager.deleteFcCs||dS)N)rA)rrEr!r!r"shutdownszCRTTransferManager.shutdowncCs$x|jD]}|s|qWdS)N)r:donerE)r coordinatorr!r!r"_cancel_transferss z$CRTTransferManager._cancel_transferscCsx|jD] }|qWdS)N)r:result)rrSr!r!r"_finish_transferss z$CRTTransferManager._finish_transferscCsx|jD] }|qWdS)N)r:%wait_until_on_done_callbacks_complete)rrSr!r!r"_wait_transfers_dones z'CRTTransferManager._wait_transfers_donecCs\|r |z@y |Wn.tk r6|Yntk rHYnXWd|XdS)N)rTrVKeyboardInterrupt ExceptionrX)rrEr!r!r"rAs   zCRTTransferManager._shutdowncKs|jdS)N)r<release)rkwargsr!r!r"_release_semaphoresz%CRTTransferManager._release_semaphorec Cs|jg}t|jd}t|j||d}tf|}t|}||yD|j|j |d}||j |||||} |j j f| } WnJtk r} z,|| d|j j |d|d} | | dWdd} ~ XYn X|| |j||jd7_|S) N) transfer_id)metarSZqueuedTrR)after_subscribers)error)r]CRTTransferCoordinatorr=CRTTransferMetaCRTTransferFutureAfterDoneHandlerappendr<acquirer9get_crt_callbackget_make_request_argsr7Z make_requestrZ set_exceptionset_s3_requestr:) r request_type call_argson_done_after_callsrS componentsfutureZ afterdoneZ on_queuedZ crt_callargsZcrt_s3_requesteon_doner!r!r"rLs2         z#CRTTransferManager._submit_transfer)N)NN)NN)NN)F)F)r(r)r*r#r@rFrNrOrPrQrTrVrXrAr]rLr!r!r!r"r4s      r4c@s>eZdZdZd ddZeddZeddZed d ZdS) rdz*Holds metadata about the CRTTransferFutureNcCs||_||_i|_dS)N) _transfer_id _call_args _user_context)rr^rnr!r!r"r#szCRTTransferMeta.__init__cCs|jS)N)ru)rr!r!r"rnszCRTTransferMeta.call_argscCs|jS)N)rt)rr!r!r"r^szCRTTransferMeta.transfer_idcCs|jS)N)rv)rr!r!r" user_context szCRTTransferMeta.user_context)NN) r(r)r*__doc__r#propertyrnr^rwr!r!r!r"rds    rdc@sDeZdZdddZeddZddZddd Zd d Zd d Z dS)reNcCs ||_|dkrt|_||_dS)aZThe future associated to a submitted transfer request via CRT S3 client :type meta: s3transfer.crt.CRTTransferMeta :param meta: The metadata associated to the transfer future. :type coordinator: s3transfer.crt.CRTTransferCoordinator :param coordinator: The coordinator associated to the transfer future. N)_metard _coordinator)rr_rSr!r!r"r#&s zCRTTransferFuture.__init__cCs|jS)N)rz)rr!r!r"r_4szCRTTransferFuture.metacCs |jS)N)r{rR)rr!r!r"rR8szCRTTransferFuture.donecCs|j|dS)N)r{rU)rtimeoutr!r!r"rU;szCRTTransferFuture.resultcCs|jdS)N)r{rE)rr!r!r"rE>szCRTTransferFuture.cancelcCs$|std|jj|dddS)z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)overrideN)rRrr{rk)r exceptionr!r!r"rkAszCRTTransferFuture.set_exception)NN)N) r(r)r*r#ryr_rRrUrErkr!r!r!r"re%s    rec@seZdZddZdS)BaseCRTRequestSerializercCs tddS)alSerialize CRT HTTP requests. :type transfer_type: string :param transfer_type: the type of transfer made, e.g 'put_object', 'get_object', 'delete_object' :type future: s3transfer.crt.CRTTransferFuture :rtype: awscrt.http.HttpRequest :returns: An unsigned HTTP request to be used for the CRT S3 client zserialize_http_request()N)NotImplementedError)r transfer_typerqr!r!r"serialize_http_requestKs z/BaseCRTRequestSerializer.serialize_http_requestN)r(r)r*rr!r!r!r"rJsrc@sVeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ ddZ dS)BotocoreCRTRequestSerializerNcCsl||_|dkri}||||jf||_|jjjd|j|jjjd|j|jjjd|j dS)aSerialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to `Session.create_client()` when serializing the request. :type session: botocore.session.Session :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*) _session_resolve_client_configZ create_client_clientr_eventsregister_capture_http_request+_change_response_to_serialized_http_request_make_fake_http_response)rsession client_kwargsr!r!r"r#[s      z%BotocoreCRTRequestSerializer.__init__cCsPd}|r|}d|kr$|d}ttd}|r<||}||d<d|d<dS)Nconfig)Zsignature_versionZs3Z service_name)Zget_default_client_configrrmerge)rrrZuser_provided_configZ client_configr!r!r"rts  z3BotocoreCRTRequestSerializer._resolve_client_configc Cst|j}|j}|jr$d||jf}g}xD|jD]6\}}t|trV|||fq4||t|dfq4Wt j |}d}|j rt |j dr|j }n t|j }t j j|j|||d} | S)Nz%s?%szutf-8seek)methodpathheadersZ body_stream)rurlrqueryritems isinstancestrrgawscrthttpZ HttpHeadersbodyhasattrrZ HttpRequestr) rZ aws_request url_partsZcrt_pathZ headers_listnamevalueZ crt_headersZcrt_body_stream crt_requestr!r!r"_crt_request_from_aws_requests*     z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestcCsT||}|jddkr4t|j}|jd|j|jddk rP|jd|S)Nhostz Content-MD5)rrgetrrsetnetlocremove)rbotocore_http_requestrrr!r!r"_convert_to_crt_http_requests  z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestcKs||jd<dS)N http_request)context)rrequestr\r!r!r"rsz2BotocoreCRTRequestSerializer._capture_http_requestcKs|d}||d<dS)Nr HTTPRequest)prepare)rrparsedr\rr!r!r"rszHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestcKstjdditdS)N)botocoreZ awsrequestZ AWSResponseFakeRawResponse)rrr\r!r!r"rs z5BotocoreCRTRequestSerializer._make_fake_http_responsecCs&t|j|f|j|jd|jdS)N)ZBucketZKeyr)getattrrrGrHrJ)rZ client_methodrnr!r!r"_get_botocore_http_requests  z7BotocoreCRTRequestSerializer._get_botocore_http_requestcCs|||jj}||}|S)N)rr_rnr)rrrqrrr!r!r"rs  z3BotocoreCRTRequestSerializer.serialize_http_request)N) r(r)r*r#rrrrrrrrr!r!r!r"rZs   rc@seZdZdddZdS)rNccs x||}|sP|VqWdS)N)read)ramtdecode_contentchunkr!r!r"streams  zFakeRawResponse.stream)rN)r(r)r*rr!r!r!r"rsrc@sdeZdZdZdddZeddZddZdd d Zdd d Z ddZ dddZ ddZ ddZ dS)rcz-A helper class for managing CRTTransferFutureNcCs0||_||_t|_d|_d|_t|_dS)N) r^ _s3_requestrrr _exception _crt_futureEvent _done_event)rr^ s3_requestr!r!r"r#s  zCRTTransferCoordinator.__init__cCs|jS)N)r)rr!r!r"rsz!CRTTransferCoordinator.s3_requestcCs|jdS)N)rr)rr!r!r"set_done_callbacks_completesz2CRTTransferCoordinator.set_done_callbacks_completecCs|j|dS)N)rwait)rr|r!r!r"rWszZos_utilsr!r!r"r#szS3ClientArgsCreator.__init__c Csd}d}tt|tj}g} |tjkrV|j} |j| }t|| ||j} | | n&|tj kr||j}|j |} | |j d<|j ||} | |||||d| |||ddS)NZ ContentLengthrRprogress)rtype recv_filepath send_filepathrsZ on_progress)rrupperDEFAULTZ GET_OBJECTrIrZget_temp_filenameRenameTempFileHandlerrgZ PUT_OBJECTZ get_file_sizerJrrri)rrmrnrSrqrorrZs3_meta_request_typeZon_done_before_callsZfinal_filepathZfile_ondone_callZdata_lenrr!r!r"rjs6      z)S3ClientArgsCreator.get_make_request_argsNcsfdd}|S)Ncsfg}dk r|7}|t7}dk r2|7}x.|D]&}dkrT||ddq8|||q8WdS)Nrr)Zbytes_transferred)r)rDr\Zcallbacks_listcallback)r`before_subscribers callback_typerqr!r"invoke_all_callbacks.s zBS3ClientArgsCreator.get_crt_callback..invoke_all_callbacksr!)rrqrrr`rr!)r`rrrqr"ri+sz$S3ClientArgsCreator.get_crt_callback)NN)r(r)r*r#rjrir!r!r!r"r8s$r8c@seZdZddZddZdS)rcCs||_||_||_||_dS)N)r{_final_filename_temp_filenamer6)rrSZfinal_filenameZ temp_filenamer?r!r!r"r#BszRenameTempFileHandler.__init__c Kst|d}|r|j|jnTy|j|j|jWn<tk rn}z|j|j|j|Wdd}~XYnXdS)Nra)r6Z remove_filerZ rename_filerrZr{rk)rr\rarrr!r!r"r'HszRenameTempFileHandler.__call__N)r(r)r*r#r'r!r!r!r"rAsrc@seZdZddZddZdS)rfcCs ||_dS)N)r{)rrSr!r!r"r#WszAfterDoneHandler.__init__cKs|jdS)N)r{r)rr\r!r!r"r'ZszAfterDoneHandler.__call__N)r(r)r*r#r'r!r!r!r"rfVsrf)9loggingiorrZbotocore.awsrequestrZbotocore.sessionrZbotocore.configrZbotocore.compatrZbotocore.exceptionsrZ awscrt.httprZ awscrt.s3rrr Z awscrt.ior r r r rZ awscrt.authrrZs3transfer.exceptionsrZs3transfer.futuresrrZs3transfer.utilsrrrZs3transfer.constantsrr getLoggerr(loggerrr3objectr4rdrerrrrcr8rrfr!r!r!r" sH         I%k 5>