B Äh·di½ã @s„ddlZddlZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZddlZddlZddlZddlZddlZddlZddlmZmZddlmZd d id d id d id d id d id d id d id d id d id d id œ ZGd d„de ƒZ!Gdd„de ƒZ"dddgZ#dZ$dd„Z%Gdd„dƒZ&Gdd„dej'ƒZ(dS)éN)Úyaspin)ÚSpinners)ÚMultiDataModel)ÚServerlessInferenceConfig)ÚModel)ÚListÚOptional)Ú PredictorZTS_DEFAULT_WORKERS_PER_MODELÚ1ZMMS_DEFAULT_WORKERS_PER_MODEL) z!huggingface-text2text-flan-t5-xxlz&huggingface-text2text-flan-t5-xxl-fp16z*huggingface-text2text-flan-t5-xxl-bnb-int8z huggingface-text2text-flan-t5-xlz#huggingface-text2text-flan-t5-largez#huggingface-text2text-flan-ul2-bf16z%huggingface-text2text-bigscience-t0ppz*huggingface-text2text-bigscience-t0pp-fp16z.huggingface-text2text-bigscience-t0pp-bnb-int8z8huggingface-textgeneration2-gpt-neoxt-chat-base-20b-fp16c@seZdZdd„Zddd„ZdS)ÚutilscCsdS)N©)Úselfr r úQ/home/ec2-user/SageMaker/easy-amazon-sagemaker-deployments/ezsmdeploy/__init__.pyÚ__init__+szutils.__init__útask == text2textcCs0ddlm}||d}tdƒtd |¡ƒdS)Nr)Úlist_jumpstart_models)Úfilterz)List of foundation models in Jumpstart: Ú )Z"sagemaker.jumpstart.notebook_utilsrÚprintÚjoin)r Z filter_valuerZtext_generation_modelsr r rÚlist_foundation_models-s  zutils.list_foundation_modelsN)r)Ú__name__Ú __module__Ú __qualname__rrr r r rr *sr c@s²eZdZd4d d „Zd d„Zdd„Zd5ejdœdd„Zdd„Z dd„Z dd„Z dd„Z dd„Z dd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)d*„Zd+d,„Zd6d.d/„Zd0d1„Zd2d3„ZdS)7ÚDeployNFéèéé TÚéédÚ*c$ Csädddddg|_dgdgddgdgdœ|_| |_||_||_||_||_||_||_| |_ ||_ d |_ ||_ ||_ ||_||_||_||_|ržt| | d |_nd|_||_t d d ¡}i|_t|d d4}t |¡}x"|D]}t|dƒ|j|d<qÚWWdQRX|jdkrd |_d|_n¸|jt|j ¡ƒks>d|jkr¾|jdkr¾d|jkrˆ|jdkrzd|_d |_|jdkn d |_d|_n4|j  d¡dddkr°d|_d |_n d |_d|_nt!dt|j ¡ƒdƒ‚t"|ƒt#krò|g|_$d |_nTt"|ƒtkr||_$d|_n8|dkr>tddƒ}WdQRXdg|_$d |_nt!dƒ‚|j sØ|jsØ|dd…dkrzt!d|dƒ‚n||_%|j%} t| ƒ$}!t& '|! (¡¡}"dd „|"j)Dƒ}#WdQRXd!|#krØd"|#krØt!d#|ƒ‚|dkrðt*j+ ,¡|_+n||_+|dkrt- .¡ /¡|_0n@t"|ƒt#kr<||_0| 1¡d krPt!d$ƒ‚nt!d%t#t"|ƒƒƒ‚| dkrh|j+ 2¡|_3n| |_3||_4|dkrž||jkrž||_5|j||_4n.|dkrÌ||jkrÌ|j sÌ|jsÌt!d&ƒ‚||_6| |_| 7¡dS)'NÚ tensorflowÚpytorchÚmxnetÚsklearnÚ huggingfaceZtorchZgluon)r"r#r$r%F)Zmemory_size_in_mbZmax_concurrencyÚ ezsmdeployz data/cost.csvÚr)ÚmoderrTÚlocalÚ local_gpuÚ.)ÚpÚgz!Please choose an instance type inz…, or choose local for local testing. Don't pass in any instance or pass in None if you want to automatically choose an instance type.ÚtmpmodelÚwzêmodel must be a single serialized file (like 'model.pkl') or a list of files ([model.pkl, model2.pkl]). If you are downloading a model in the script or packaging with the container, pass in model = NoneéþÿÿÿÚpyz9please provide a valid python script with .py extension. z is invalidcSsg|]}t|tjƒr|j‘qSr )Ú isinstanceÚastÚ FunctionDefÚname)Ú.0Únr r rú Ósz#Deploy.__init__..Z load_modelÚpredictz›please implement a load_model(modelpath) that returns a loaded model, and predict(inputdata) function that returns a prediction in yourz¥please enter a name with lower case letters; we will be using this name for s3 bucket prefixes, model names, ECR repository names etc. that have various restrictionszKenter string for a name or don't pass in a name; type of name passed in is zIf requirements=None, please provide a value for framework; choice should be one of 'tensorflow','pytorch','mxnet','sklearn')8Z frameworklistZframeworkinstallsÚwaitÚbudgetÚinstance_countÚ instance_typeÚimageÚdockerfilepathÚeiÚprefixÚmonitorÚdeployedÚautoscaletargetÚfoundation_modelÚfoundation_model_versionÚhuggingface_modelÚhuggingface_model_taskÚ serverlessrÚserverless_configÚhuggingface_model_quantizeÚ pkg_resourcesÚresource_filenameÚcostdictÚopenÚcsvÚreaderÚfloatÚgpuÚ multimodelÚlistÚkeysÚsplitÚ ValueErrorÚtypeÚstrÚmodelÚscriptr4ÚparseÚreadÚbodyÚ sagemakerÚsessionÚSessionÚ shortuuidÚuuidÚlowerr6ÚislowerZdefault_bucketÚbucketÚ requirementsÚ frameworkÚ autoscaleÚdeploy)$r r\r]rjrir6rkrErJZserverless_memoryZserverless_concurrencyr;rhrBrbr?r@r>r=r<rArCrFrGrHrIrLZcostpathÚinfilerRÚrowsÚfpÚfilenameÚfileÚnodeÚ functionsr r rr5sÔ    $               &zDeploy.__init__c CsÜ|jdkr|jstdƒ‚ddlm}m}|jd|_d|ji}|jdk rR|j|d<|jsxyVt  d¡}|j |j  d¡gdd d}d |kr¬t   |d d dd ¡|d <nWntk r@}znt|ddtdƒt   d¡|d <d|jkrVt   d¡|d <d|jkrt   d¡|d <n6d|jkr:t   d¡|d <nd|jkr0t   d¡|d <nÚd|jkr¨t   d¡|d <d|jkrŒt   d¡|d <nd|jkr0t   d¡|d <nˆd|jkrt   d¡|d <d|jkrÞt   d¡|d <n6d|jkrút   d¡|d <nd|jkr0t   d¡|d <nd|jkr0t   d¡|d <Wdd}~XYnX|jdkrZ|j|d<n|jdkrhntd |j›ƒ‚t ¡}d!|j}} |jsÄ|jrÄ|jrÄ||d"ƒ|||d#d$d%d&|_n|||d#d$d%d'|_dS)(Nz.Please enter a valid instance type, not [None]r)ÚHuggingFaceModelÚget_huggingface_llm_image_uriZ HF_MODEL_IDZHF_TASKZec2zml.)Ú InstanceTypesrvZGpuInfoZGpusZCountZ SM_NUM_GPUSz ... )ÚendzNTrying fallback to figure out number of GPUs in the instance type you chose - r.rZ12xéÚ24xÚ48xÚp2Z8xéÚ16xéÚp3Úp4)Z bitsandbytesZgptqZHF_MODEL_QUANTIZEzFhuggingface_model_quantize needs to be one of bitsandbytes, gptq, not z hf-model-r&z4.26z1.13Zpy39)Ú image_uriÚenvÚroler6Útransformers_versionÚpytorch_versionÚ py_version)r‚rƒr„r…r†)r>rJrYZsagemaker.huggingfacertrur\rIÚboto3ÚclientZdescribe_instance_typesÚstripÚjsonÚdumpsÚ ExceptionrrLraÚget_execution_roler6rFrHÚsagemakermodel) r rtruZhubZ ec2_clientÚrespÚeÚaws_roleÚ endpoint_namer6r r rÚdeploy_huggingface_model s€                        zDeploy.deploy_huggingface_modelc Csöddlm}m}m}m}|jd|_|j|j|jdd}|jdkrH||_tjj ddd|j|j|d}tjj |j|jdd}tjj |j|jdd}t  ¡} d|j } |jt krÒtjj ||| tjj| t |jd |_n dd lm} | |j|j| d |_dS) Nr)Ú image_urisÚinstance_typesÚ model_urisÚ script_urisÚ inference)Úmodel_idÚ model_versionÚscope)ÚregionrjZ image_scoper™ršr>)r™ršZ script_scope)r™ršZ model_scopezmodel-)rÚ model_datarƒÚ predictor_clsr6r‚)ÚJumpStartModel)r™ršrƒ)rar”r•r–r—r\Zretrieve_defaultrGr>Úretrieverr6Ú_model_env_variable_maprÚ predictorr rŽZsagemaker.jumpstart.modelrŸ) r r”r•r–r—r>Zdeploy_image_uriZdeploy_source_uriZ model_urir‘r’rŸr r rÚdeploy_foundation_modelds>     zDeploy.deploy_foundation_modelÚsrc)ÚreturncCs6tjdt ¡jd}tjj|d}tjj||d}|S)zReturn the SageMaker session.ra)Z service_nameÚ region_name)Úlocal_download_dir)Úsagemaker_clientÚsettings)r‡rˆrcr¦raÚsession_settingsZSessionSettingsrb)r r§r¨rªrbr r rÚget_sagemaker_sessionžs zDeploy.get_sagemaker_sessionc Cs@i|_|jdkr²t dd¡}t|ddz}t |¡}xh|D]`}t|dƒdt|dƒ|j|d|j|dt|dƒdt|dƒf|j|d<q:WWdQRX|  ¡nŠ|jt |j  ¡ƒksÒ|jdkr&|jdkr|j|j|_ |j dkr$d d d d d ddœ}|j ||j |_ nd|_ ntdt |j  ¡ƒdƒ‚dS)Nr'zdata/instancetypes.csvr()r)érr)r*r+g¸…ëQ¸¾?g¸…ëQ¸Î?gÃõ(\ÂÕ?g¤p= ×£À?g¤p= ×£Ð?g¤p= ×£à?)zml.eia2.mediumz ml.eia2.largezml.eia2.xlargez ml.eia.mediumz ml.eia.largez ml.eia.xlargez!Please choose an instance type inz$, or choose local for local testing.)Ú instancedictr>rMrNrPrQrRrSrOÚchoose_instance_typerVrWÚ costperhourrArY)r ZinstancetypepathrmrRrnZeicostsr r rÚprocess_instance_type°s8    D     zDeploy.process_instance_typec CsL|jdkrd|jd}n|jd|jd}| |j|¡}t dd¡|_d}d}x|t|j  ¡ƒD]j}|j|d}|j|d }|j|d }|j d kr´||krÌ||krÌ|}|}qb||krb||j krb|}PqbW|dkr |j d kr t d t |j ƒd t |ƒdƒ‚n*|dkr4|j d kr4t dt |ƒdƒ‚||_ |j|j |_dS)Nrzezsmdeploy/model-ú/z/ezsmdeploy/model-r'zdata/instancetypes.csvièrrr¬r z9Could not find an instance that satisfies your budget of z8 per hour and can host your models with a total size of z, Gb. Please choose a higher budget per hour.z3You may be using large models with a total size of z` Gb. Please choose a high memory GPU instance and launch without multiple models (if applicable))rBr6Úget_sizerhrMrNZinstancetypespathrVr­rWr<rYr[r>rOr¯) r ÚtmppathÚsizeZchoseninstanceZmincostÚinstanceZ memperworkerZcostZ costpermemr r rr®ás6   "zDeploy.choose_instance_typecCs|j ||¡dS)N)rŽÚ add_model)r Zs3pathZ relativepathr r rr¶szDeploy.add_modelcCsš|js2td|j|jd|jt ¡tjjd|_ ndt d|jd  |jd  d¡dd…¡d|jt ¡tjjd|_ x|jD]}|  |d¡q|Wd|_dS) Nzmodel-r)r6rrrƒržr±éÿÿÿÿ)r6Zmodel_data_prefixrrƒržzserving/F)rUrr6Ú modelpathr?rarr¢r rŽrrrXr¶rA)r Úpathr r rÚ create_model s   zDeploy.create_modelc Cs˜|jrPddlm}tdkr,d |j|j¡}nd |j|j|j¡}|dd|d}nd}|jdk r®d}d |jksšd |jksšd |jksšd |jksšd |jksš|jržd}d|jkr²d}nd}|jrä|j sä|j j |j |jd|j||j d|_n¤|jr|j r|j j |j |jd|j||j dd|_nl|j rV|jsV|j j |j |jd|j||j|j d|_n2|j j |j |j|jd|jd|j |||jdd |_d|j|_dS)Nr)ÚDataCaptureConfigrz's3://{}/ezsmdeploy/model-{}/datacapturez*s3://{}/{}/ezsmdeploy/model-{}/datacaptureTr )Zenable_captureZsampling_percentageZdestination_s3_urirr€r}ryrzéZg5zezsm-foundation-endpoint-)Úinitial_instance_countr>r’Ú volume_sizer;zezsm-hf-endpoint-i,)r½r>r’r¾r;Ú&container_startup_health_check_timeout)r½r>r’r¾Úserverless_inference_configr;zezsm-endpoint-F) r½r>Zaccelerator_typer’Zupdate_endpointr;r¾Údata_capture_configrÀr¿)rCZsagemaker.model_monitorr»rBÚformatrhr6r>rFrHrŽrlr=r;r¢rKrAr’)r r»Ztmps3urirÁr¾r r rÚ deploy_model=sl   8    zDeploy.deploy_modelcCsBt d¡}| |¡}d}x |jj|dD]}||j}q(W|dS)NÚs3g)ZPrefixgÐA)r‡ÚresourceZBucketÚobjectsrr´)r rhr¹rÄZ my_bucketZ total_sizeÚobjr r rr²–s   zDeploy.get_sizecCshd}|jdkrd}n |jd}g|_x>|jD]4}|j |jjd |¡|j||jd¡|d7}q,WdS)Nrrzezsmdeploy/model-z/ezsmdeploy/model-zmodel{}.tar.gz)r¹rhÚ key_prefix) rBr¸r\ÚappendrbZ upload_datarÂrhr6)r Úir³r6r r rÚ upload_model s   zDeploy.upload_modelc Cs`d}xT|jD]H}d|krœd|krœ|jjd |¡| d¡dd | d¡dd…¡dt t d  |¡¡d ¡}|  d  |¡¡WdQRXd  |¡}nvd|krd|kr|  d  |¡¡t   |d  |¡¡t t d  |¡¡d ¡}|  d  |¡¡WdQRXd  |¡}t d |¡d¡}d|kr>|j |ddn |  |¡| ¡|d7}qWdS)Nrztar.gzrÄz./downloads/{}r±r¬é)r¹rhrÈz./downloads/{}/*.tar.gzrz./extractedmodel/{}/zextractedmodel/{}/z./downloads/{}/zmodel{}.tar.gzzw:gzr,)Úarcname)r\rbZ download_datarÂrXrÚtarfilerPÚglobÚ extractallÚ makedir_safeÚshutilÚcopyÚaddÚclose)r rÊr6Útarr r rÚ tar_model±s4     zDeploy.tar_modelc Cstyt |¡Wn YnXytj |¡s4t |¡Wn8tk rn}z|jdkr^t|jƒ‚Wdd}~XYnXdS)Né) rÒÚrmtreeÚosr¹ÚexistsÚmakedirsÚOSErrorÚerrnor)r Ú directoryÚerrr r rrÑÙs   zDeploy.makedir_safecCsŠ| d¡t|jƒtkrBtj |j¡r6t |jd¡q†|jd‚nDt|jƒt kr~t ddƒ}t dd„|jƒ}|  |¡|  ¡ntdƒ‚dS)Nr¤zsrc/requirements.txtz does not exist!r0cSs|dS)Nrr )Úxr r rÚ÷óz,Deploy.handle_requirements..zUpass in a path/to/requirements.txt or a list of requirements ['scikit-learn',...,...])rÑrZrir[rÚr¹rÛrÒrÓrVrPÚmapÚ writelinesrÕrY)r ÚfÚl1r r rÚhandle_requirementsès     zDeploy.handle_requirementsc Csšd}tddƒ}tj| |j¡|dd}WdQRXt d¡ ¡ d¡d}t d ¡ ¡ d¡d}d  |||j¡|_ xtj   d ¡sŽt   d ¡qvWd |_dS)Nz=chmod +x src/build-docker.sh & sudo ./src/build-docker.sh {}zsrc/dockeroutput.txtr0T)ÚstdoutÚshellz9aws sts get-caller-identity --query Account --output textrrzaws configure get regionz/{}.dkr.ecr.{}.amazonaws.com/ezsmdeploy-image-{}z src/done.txtrÌzPlease see src/dockeroutput.txt)rPÚ subprocessÚPopenrÂr6rÚÚpopenr_rXr?r¹rÛÚtimeÚsleepZ dockeroutput)r Úcmdrær-Zacctrœr r rÚ build_dockers " zDeploy.build_dockerc Cs”t d¡j|jd}|d}|ddd}t d¡}|jdd ||¡d d d d }|jd  |j¡dd ||¡d d|jddiddddœd}||_ dS)Nra)Ú EndpointNameròZProductionVariantsrZ VariantNamezapplication-autoscalingzendpoint/{}/variant/{}z&sagemaker:variant:DesiredInstanceCountrr)ÚServiceNamespaceÚ ResourceIdÚScalableDimensionZ MinCapacityZ MaxCapacityzscaling-policy-{}ZTargetTrackingScalingZPredefinedMetricTypeZ&SageMakerVariantInvocationsPerInstanceiXF)Z TargetValueZPredefinedMetricSpecificationZScaleOutCooldownZScaleInCooldownZDisableScaleIn)Z PolicyNamerórôrõZ PolicyTypeZ(TargetTrackingScalingPolicyConfiguration) r‡rˆZdescribe_endpointr’Zregister_scalable_targetrÂZput_scaling_policyr6rEZscalingresponse)r ÚresponseZin1Zin2rˆr r rÚautoscale_endpoints.       zDeploy.autoscale_endpointéc Cst|jr|dkrtdƒ‚|jrht dd¡}t |d¡tj ¡}t t j ddd}|  ¡|  ttj ¡|ƒd¡| ¡|jr²td d ƒ} t |jd d œ| ¡WdQRXn*td d ƒ} t |jdd œ| ¡WdQRXt |td dƒ¡d |||¡} t | ¡} xtj d¡s$t d¡qWt d¡|  ¡|  ttj ¡|ƒd¡| ¡WdQRXntdƒ‚dS)NzXsince this is a multimodel endpoint, please pass in a target model that you wish to testr'zdata/smlocust.pyzsrc/smlocust.pyÚgreenr)ÚcolorÚtextz | Starting test with Locustzsrc/locustdata.txtr0z model1.tar.gz)r’Ú target_modelzsrc/testdata.pÚwbzklocust -f src/smlocust.py --no-web -c {} -r {} --run-time {}s --csv=src/locuststats; touch src/testdone.txtzsrc/testdone.txtrÌz9 | Done! Please see the src folder for locuststats* filesz-Deploy model to endpoint first before testing)rUrYrDrMrNrÒrÓÚdatetimeÚnowrrÚpointÚhideÚwriter[ÚshowrPrŠÚdumpr’ÚpicklerÂrÚÚsystemr¹rÛrîrïÚremove) r Z input_datarüZ usercountZ hatchrateZ timeoutsecsÚpath1ÚstartÚspÚoutfilerðr-r r rÚtest>sH         z Deploy.testc CsÒtj ¡}ttjddd¬}|jsÌ|jsÌyt d¡Wn YnX|  ¡|  ¡|j dgkr„|  t tj ¡|ƒd¡n|  t tj ¡|ƒd¡| ¡| ¡| ¡|  ¡|  t tj ¡|ƒd¡| ¡|jdkrt tj ¡|ƒd }| d ¡n| ¡t tj ¡|ƒd }|  ¡|  |¡| ¡t |jd ¡|  ¡|  t tj ¡|ƒd ¡| ¡|jdkr¬|jdkr¬t dd¡|_n&|jdkrÒ|jdkrÒt dd¡|_t |jd¡|  ¡|  t tj ¡|ƒd¡| ¡|jrdt dd¡}t dd¡}t dd¡}t |d¡t |d¡t |d¡d|_nÖt dd¡}t dd¡}t dd¡}t dd¡}t dd¡}t dd¡} t |d ¡t |d!¡t |d"¡t |d#¡t |d$¡t | d¡|jr:|jdkr:d|_|  ¡|  t tj ¡|ƒd%¡| ¡|  ¡|  t tj ¡|ƒd&¡| ¡|jdkrÂ|  t tj ¡|ƒd'¡| ¡|  ¡|  t tj ¡|ƒd(¡| ¡| ¡n4|jræ|jsæ| ¡n|jrø|  ¡nt!d)ƒ‚|  ¡|j"s4|  t tj ¡|ƒd*|j#¡n|  t tj ¡|ƒd+¡| ¡| $¡|  ¡|  t tj ¡|ƒd,¡| ¡|j%rÖ|j#d-krÖ| &¡|  ¡|  t tj ¡|ƒd.¡| ¡n@|j%r|j#d-kr|  ¡|  t tj ¡|ƒd/¡| ¡|j'rZ|  ¡|  t tj ¡|ƒd0d1 (|j)|j*¡¡| ¡|j+ ,t tj ¡|ƒd2¡d|_-y6t. /d3¡t. /d ¡t. /d4¡t. /d5¡t. /d¡Wn YnX|j0SQRXdS)6Nrùr)rúrûzsrc/r/z^ | No model was passed. Assuming you are downloading a model in the script or in the containerz | compressed model(s)z7 | uploaded model tarball(s) ; check returned modelpathz# | no additional requirements foundr¤z | added requirements filezsrc/transformscript.pyz | added source fileTr'zdata/DockerfileFzdata/Dockerfile_flaskzsrc/Dockerfilez | added Dockerfilezdata/model_handler.pyzdata/dockerd-entrypoint.pyzdata/build-docker.shzsrc/model_handler.pyzsrc/dockerd-entrypoint.pyzsrc/build-docker.shzdata/nginx.confzdata/predictor.pyz data/servez data/trainz data/wsgi.pyzsrc/nginx.confzsrc/predictor.pyz src/servez src/trainz src/wsgi.pyz^ | Setting Elastic Inference to None since you selected a GPU instancez' | added model_handler and docker utilsz | building docker containerz | built docker containerzrÃrkr÷rCrÂrhr6rùÚokrDrÚrr¢) r r r ZrtextrZpath2Zpath3Zpath4Zpath5Zpath6r r rrl|s                                z Deploy.deploycCs t|j|jddddd ¡dS)Né€Tg333333ã?é()r¢Ú model_nameÚmax_new_tokensZ do_sampleZ temperatureZtop_k)ÚOpenChatKitShellr¢r\Úcmdloop)r r r rÚchatdsz Deploy.chat)NNNNFrFrrTNrNNNNrr NFFr!FNN)r¤)Nrrørø)rrrrr“r£rarcr«r°r®r¶rºrÃr²rËr×rÑrèrñr÷r rlrr r r rr4sZ <Y:1<Y (% =irzzz <|endoftext|>z#Current Date: {} Current Time: {} cCs(xtD]}| |d¡}qW| d¡}|S)Nrr)ÚMEANINGLESS_WORDSÚreplacer‰)röÚwordr r rÚclean_response{s  rc@sHeZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Ze d d„ƒZ dS)Ú ConversationcCs2t d¡}t d¡}||_||_t ||¡|_dS)Nz%Y-%m-%dz%H:%M:%S %p %Z)rîÚstrftimeÚ _human_idÚ_bot_idÚ PRE_PROMPTrÂÚ_prompt)r Úhuman_idÚbot_idZcur_dateÚcur_timer r rr‚s   zConversation.__init__cCs |j|j›d|›d7_dS)Nz: r)rr)r Úcontextr r rÚpush_context_turnŠszConversation.push_context_turncCs6|j|j›d|›d7_|j|j›d7_dS)Nz: rú:)rrr)r Úqueryr r rÚpush_human_turnŽszConversation.push_human_turncCsL|j|k}| |j›d¡d}t|ƒ}|s4|d7}|j|›d7_dS)Nr$rz...r)rrXrr)r röZ has_finishedZbot_turnr r rÚpush_model_response’s  z Conversation.push_model_responsecCs<|j›d}|j›d}t d|›d|›d|j¡}|dS)Nr$ú(ú|z)\W?r·)rrÚrerXr)r Z human_tagZbot_tagZturnsr r rÚ get_last_turnœs  zConversation.get_last_turncCs|jS)N)r)r r r rÚget_raw_prompt¢szConversation.get_raw_promptcCs |t_dS)N)r r)ÚclsÚvaluer r rÚfrom_raw_prompt¥szConversation.from_raw_promptN) rrrrr#r&r'r+r,Ú classmethodr/r r r rrs rcspeZdZdZdZdZdZdeee e edœ‡fdd„ Z d d „Z d d „Z d d„Zdd„Zdd„Zdd„Z‡ZS)rzjEzSMdeploy Openchatkit shell - Type /help or /? to list commands. For example, type /quit to exit shell. z>>> zzN)r¢rÚ cmd_queuec s<tƒ ¡||_||_||_|jg|jd<|dk r8||_dS)NZstopping_criteria)Úsuperrr¢r\Úpayload_kwargsrÚcmdqueue)r r¢rr1Úkwargs)Ú __class__r rr´s zOpenChatKitShell.__init__cCst|j|jƒ|_dS)N)rrr Ú conversation)r r r rÚpreloop½szOpenChatKitShell.preloopcCs"| d¡r|dd…nd|}|S)Nr±rzsay )Ú startswith)r ÚlineÚcommandr r rÚprecmdÀszOpenChatKitShell.precmdcCsÔ|j |¡|j ¡}d|jkrXd|i|j–}|j |¡}|dddt|ƒd…}n^d|jksld|jkr²|ddid œ}|j |¡}|dd d ¡}|ddt|ƒ|…}nd }|j  |¡t |j  ¡ƒdS) Nz neoxt-chatZ text_inputsrZgenerated_textz-Chatrrr )ÚinputsÚ parametersz z1I don't recognize the output from this chat model) r7r&r,r\r3r¢r:ÚlenÚrfindr'rr+)r ÚargÚpromptÚpayloadröÚoutputÚlastr r rÚdo_sayÄs       zOpenChatKitShell.do_saycCst|j|jƒ|_dS)N)rrr r7)r rAr r rÚdo_resetÜszOpenChatKitShell.do_resetcCstd|j›dƒdS)NzHyperparameters: r)rr3)r rAr r rÚdo_hyperparametersßsz#OpenChatKitShell.do_hyperparameterscCsdS)NTr )r rAr r rÚdo_quitâszOpenChatKitShell.do_quit)N)rrrÚintrorBrr r r[rrrr8r<rFrGrHrIÚ __classcell__r r )r6rr«s r))rardrZyaspin.spinnersrrîrþrÎr*r‡rÏrÚrÒrMrëZsagemaker.multidatamodelrZsagemaker.serverlessrZsagemaker.modelrr4rQrŠrrðÚtypingrrZsagemaker.predictorr r¡Úobjectr rrrrrÚCmdrr r r rÚsb        H *