B 劇cV@sddlZddlZddlmZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZddlmZmZmZddlmZddlmZddlmZdd lmZdd lmZmZdd l m!Z!dd l"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(e)e*Z+GdddZ,dddeddeddfddZ-GdddZ.Gddde$Z/Gddde#Z0GdddZ1Gddde1Z2Gdd d eZ3Gd!d"d"Z4Gd#d$d$Z5Gd%d&d&Z6Gd'd(d(Z7dS))N)BytesIO)AwsCredentialsAwsCredentialsProvider)ClientBootstrapClientTlsContextDefaultHostResolverEventLoopGroupTlsContextOptions)S3ClientS3RequestTlsMode S3RequestType)UNSIGNED)urlsplit)Config)NoCredentialsError)GBMB)TransferNotDoneError)BaseTransferFutureBaseTransferMeta)CallArgsOSUtils get_callbacksc@s$eZdZddZddZddZdS)CRTCredentialProviderAdaptercCs||_d|_t|_dS)N)_botocore_credential_provider_loaded_credentials threadingLock_lock)selfbotocore_credential_providerr!i/private/var/folders/8c/hx9_v10d5x38qmnzt13b7b8j1k3n5b/T/pip-target-x6xd5gna/lib/python/s3transfer/crt.py__init__+sz%CRTCredentialProviderAdapter.__init__cCs|}t|j|j|jS)N)_get_credentialsZget_frozen_credentialsrZ access_keyZ secret_keytoken)r credentialsr!r!r"__call__0s z%CRTCredentialProviderAdapter.__call__c Cs@|j0|jdkr0|j}|dkr*t||_|jSQRXdS)N)rrrZload_credentialsr)rZ loaded_credsr!r!r"r$6s  z-CRTCredentialProviderAdapter._get_credentialsN)__name__ __module__ __qualname__r#r'r$r!r!r!r"r*srTc Cst|}t|}t||} d} d} |r,tjntj} |dk rht} |rR| j|dnd| _t | }| } |r~t |}t |} |dt}t| || || | |dS)a :type region: str :param region: The region used for signing :type botocore_credential_provider: Optional[botocore.credentials.CredentialResolver] :param botocore_credential_provider: Provide credentials for CRT to sign the request if not set, the request will not be signed :type num_threads: Optional[int] :param num_threads: Number of worker threads generated. Default is the number of processors in the machine. :type target_throughput: Optional[int] :param target_throughput: Throughput target in Bytes. Default is 0.625 GB/s (which translates to 5 Gb/s). :type part_size: Optional[int] :param part_size: Size, in Bytes, of parts that files will be downloaded or uploaded in. :type use_ssl: boolean :param use_ssl: Whether or not to use SSL. By default, SSL is used. Note that not all services support non-ssl connections. :type verify: Optional[boolean/string] :param verify: Whether or not to verify SSL certificates. By default SSL certificates are verified. You can provide the following values: * False - do not validate SSL certificates. SSL will still be used (unless use_ssl is False), but SSL certificates will not be verified. * path/to/cert/bundle.pem - A filename of the CA cert bundle to use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. N)Z ca_filepathFr,) bootstrapregionZcredential_provider part_sizetls_modetls_connection_optionsZthroughput_target_gbps)rrrr ZENABLEDZDISABLEDr Z&override_default_trust_store_from_pathZ verify_peerrZnew_connection_optionsrrZ new_delegaterr )r.r Z num_threadsZtarget_throughputr/Zuse_sslverifyZevent_loop_groupZ host_resolverr-providerr1r0Ztls_ctx_optionsZclient_tls_optionZcredentails_provider_adapterZ target_gbpsr!r!r"create_s3_crt_clientBs8/   r4c@seZdZdddZddZddZddd Zdd d Zd d d Zd!ddZ ddZ ddZ ddZ d"ddZ ddZddZdS)#CRTTransferManagerNcCs@|dkrt|_||_t||j|_g|_td|_d|_ dS)aFA transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client :param crt_s3_client: The CRT s3 client, handling all the HTTP requests and functions under then hood :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer :param crt_request_serializer: Serializer, generates unsigned crt HTTP request. :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. Nr) r_osutil_crt_s3_clientS3ClientArgsCreator_s3_args_creator_future_coordinatorsr Semaphore _semaphore _id_counter)rZ crt_s3_clientcrt_request_serializerosutilr!r!r"r#s  zCRTTransferManager.__init__cCs|S)Nr!)rr!r!r" __enter__szCRTTransferManager.__enter__cGsd}|r d}||dS)NFT) _shutdown)rexc_type exc_valueargscancelr!r!r"__exit__szCRTTransferManager.__exit__cCs6|dkr i}|dkri}t|||||d}|d|S)N)bucketkeyfileobj extra_args subscribers get_object)r_submit_transfer)rrHrIrJrKrLcallargsr!r!r"downloadszCRTTransferManager.downloadcCs6|dkr i}|dkri}t|||||d}|d|S)N)rHrIrJrKrLZ put_object)rrN)rrJrHrIrKrLrOr!r!r"uploadszCRTTransferManager.uploadcCs4|dkr i}|dkri}t||||d}|d|S)N)rHrIrKrLZ delete_object)rrN)rrHrIrKrLrOr!r!r"deleteszCRTTransferManager.deleteFcCs||dS)N)rB)rrFr!r!r"shutdownszCRTTransferManager.shutdowncCs$x|jD]}|s|qWdS)N)r;donerF)r coordinatorr!r!r"_cancel_transferss z$CRTTransferManager._cancel_transferscCsx|jD] }|qWdS)N)r;result)rrUr!r!r"_finish_transferss z$CRTTransferManager._finish_transferscCsx|jD] }|qWdS)N)r;%wait_until_on_done_callbacks_complete)rrUr!r!r"_wait_transfers_dones z'CRTTransferManager._wait_transfers_donecCs\|r |z@y |Wn.tk r6|Yntk rHYnXWd|XdS)N)rVrXKeyboardInterrupt ExceptionrZ)rrFr!r!r"rBs   zCRTTransferManager._shutdowncKs|jdS)N)r=release)rkwargsr!r!r"_release_semaphoresz%CRTTransferManager._release_semaphorec Cs|jg}t|jd}t|j||d}tf|}t|}||yD|j|j |d}||j |||||} |j j f| } WnJtk r} z,|| d|j j |d|d} | | dWdd} ~ XYn X|| |j||jd7_|S) N) transfer_id)metarUZqueuedTrT)after_subscribers)error)r_CRTTransferCoordinatorr>CRTTransferMetaCRTTransferFutureAfterDoneHandlerappendr=acquirer:get_crt_callbackget_make_request_argsr8Z make_requestr\ set_exceptionset_s3_requestr;) r request_type call_argson_done_after_callsrU componentsfutureZ afterdoneZ on_queuedZ crt_callargsZcrt_s3_requesteon_doner!r!r"rNs8         z#CRTTransferManager._submit_transfer)N)NN)NN)NN)F)F)r(r)r*r#rArGrPrQrRrSrVrXrZrBr_rNr!r!r!r"r5s     r5c@s>eZdZdZd ddZeddZeddZed d ZdS) rfz*Holds metadata about the CRTTransferFutureNcCs||_||_i|_dS)N) _transfer_id _call_args _user_context)rr`rpr!r!r"r#1szCRTTransferMeta.__init__cCs|jS)N)rw)rr!r!r"rp6szCRTTransferMeta.call_argscCs|jS)N)rv)rr!r!r"r`:szCRTTransferMeta.transfer_idcCs|jS)N)rx)rr!r!r" user_context>szCRTTransferMeta.user_context)NN) r(r)r*__doc__r#propertyrpr`ryr!r!r!r"rf.s    rfc@sDeZdZdddZeddZddZddd Zd d Zd d Z dS)rgNcCs ||_|dkrt|_||_dS)aZThe future associated to a submitted transfer request via CRT S3 client :type meta: s3transfer.crt.CRTTransferMeta :param meta: The metadata associated to the transfer future. :type coordinator: s3transfer.crt.CRTTransferCoordinator :param coordinator: The coordinator associated to the transfer future. N)_metarf _coordinator)rrarUr!r!r"r#Ds zCRTTransferFuture.__init__cCs|jS)N)r|)rr!r!r"raRszCRTTransferFuture.metacCs |jS)N)r}rT)rr!r!r"rTVszCRTTransferFuture.donecCs|j|dS)N)r}rW)rtimeoutr!r!r"rWYszCRTTransferFuture.resultcCs|jdS)N)r}rF)rr!r!r"rF\szCRTTransferFuture.cancelcCs$|std|jj|dddS)z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)overrideN)rTrr}rm)r exceptionr!r!r"rm_szCRTTransferFuture.set_exception)NN)N) r(r)r*r#r{rarTrWrFrmr!r!r!r"rgCs    rgc@seZdZddZdS)BaseCRTRequestSerializercCs tddS)alSerialize CRT HTTP requests. :type transfer_type: string :param transfer_type: the type of transfer made, e.g 'put_object', 'get_object', 'delete_object' :type future: s3transfer.crt.CRTTransferFuture :rtype: awscrt.http.HttpRequest :returns: An unsigned HTTP request to be used for the CRT S3 client zserialize_http_request()N)NotImplementedError)r transfer_typersr!r!r"serialize_http_requestjs z/BaseCRTRequestSerializer.serialize_http_requestN)r(r)r*rr!r!r!r"risrc@sVeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ ddZ dS)BotocoreCRTRequestSerializerNcCsl||_|dkri}||||jf||_|jjjd|j|jjjd|j|jjjd|j dS)aSerialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to `Session.create_client()` when serializing the request. :type session: botocore.session.Session :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*) _session_resolve_client_configZ create_client_clientraeventsregister_capture_http_request+_change_response_to_serialized_http_request_make_fake_http_response)rsession client_kwargsr!r!r"r#zs       z%BotocoreCRTRequestSerializer.__init__cCsPd}|r|}d|kr$|d}ttd}|r<||}||d<d|d<dS)Nconfig)Zsignature_versionZs3 service_name)Zget_default_client_configrr merge)rrrZuser_provided_configZ client_configr!r!r"rs  z3BotocoreCRTRequestSerializer._resolve_client_configc Cst|j}|j}|jr&|d|j}g}xD|jD]6\}}t|trX|||fq6||t|dfq6Wt j |}d}|j rt |j dr|j }n t|j }t j j|j|||d} | S)N?zutf-8seek)methodpathheadersZ body_stream)rurlrqueryritems isinstancestrriawscrthttpZ HttpHeadersbodyhasattrrZ HttpRequestr) rZ aws_request url_partsZcrt_pathZ headers_listnamevalueZ crt_headersZcrt_body_stream crt_requestr!r!r"_crt_request_from_aws_requests*     z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestcCsT||}|jddkr4t|j}|jd|j|jddk rP|jd|S)Nhostz Content-MD5)rrgetrrsetnetlocremove)rbotocore_http_requestrrr!r!r"_convert_to_crt_http_requests   z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestcKs||jd<dS)N http_request)context)rrequestr^r!r!r"rsz2BotocoreCRTRequestSerializer._capture_http_requestcKs|d}||d<dS)Nr HTTPRequest)prepare)rrparsedr^rr!r!r"rszHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestcKstjdditdS)N)botocoreZ awsrequestZ AWSResponseFakeRawResponse)rrr^r!r!r"rs z5BotocoreCRTRequestSerializer._make_fake_http_responsecCs&t|j|f|j|jd|jdS)N)ZBucketKeyr)getattrrrHrIrK)rZ client_methodrpr!r!r"_get_botocore_http_requests z7BotocoreCRTRequestSerializer._get_botocore_http_requestcCs|||jj}||}|S)N)rrarpr)rrrsrrr!r!r"rs  z3BotocoreCRTRequestSerializer.serialize_http_request)N) r(r)r*r#rrrrrrrrr!r!r!r"rys   rc@seZdZdddZdS)rNccs x||}|sP|VqWdS)N)read)ramtdecode_contentchunkr!r!r"streams  zFakeRawResponse.stream)rN)r(r)r*rr!r!r!r"rsrc@sdeZdZdZdddZeddZddZdd d Zdd d Z ddZ dddZ ddZ ddZ dS)rez-A helper class for managing CRTTransferFutureNcCs0||_||_t|_d|_d|_t|_dS)N) r` _s3_requestrrr _exception _crt_futureEvent _done_event)rr` s3_requestr!r!r"r#s  zCRTTransferCoordinator.__init__cCs|jS)N)r)rr!r!r"rsz!CRTTransferCoordinator.s3_requestcCs|jdS)N)rr)rr!r!r"set_done_callbacks_completesz2CRTTransferCoordinator.set_done_callbacks_completecCs|j|dS)N)rwait)rr~r!r!r"rYsz.invoke_all_callbacksr!)rrsrrrbrr!)rbrrrsr"rkOsz$S3ClientArgsCreator.get_crt_callback)NN)r(r)r*r#rlrkr!r!r!r"r9&s(r9c@seZdZddZddZdS)rcCs||_||_||_||_dS)N)r}_final_filename_temp_filenamer7)rrUZfinal_filenameZ temp_filenamer@r!r!r"r#jszRenameTempFileHandler.__init__c Kst|d}|r|j|jnTy|j|j|jWn<tk rn}z|j|j|j|Wdd}~XYnXdS)Nrc)r7Z remove_filerZ rename_filerr\r}rm)rr^rcrtr!r!r"r'pszRenameTempFileHandler.__call__N)r(r)r*r#r'r!r!r!r"risrc@seZdZddZddZdS)rhcCs ||_dS)N)r})rrUr!r!r"r#szAfterDoneHandler.__init__cKs|jdS)N)r}r)rr^r!r!r"r'szAfterDoneHandler.__call__N)r(r)r*r#r'r!r!r!r"rhsrh)8loggingriorZ awscrt.httprZbotocore.awsrequestrZbotocore.sessionZ awscrt.authrrZ awscrt.iorrrrr Z awscrt.s3r r r r Zbotocore.compatrZbotocore.configrZbotocore.exceptionsrZs3transfer.constantsrrZs3transfer.exceptionsrZs3transfer.futuresrrZs3transfer.utilsrrr getLoggerr(loggerrr4r5rfrgrrrrer9rrhr!r!r!r" sF         O&o 5C