3 .`0@sbddlZddlZddlZddlZddlZddlZddlZddl Z ddl j Z ddl Z ddlZ ddl mZmZmZddlmZmZmZejeZejejejejejdZedej dddZ!d d Z"d d Z#d dZ$ddZ%ddZ&ddZ'ddZ(ddZ)edkr^ej*Z+e+j,de-dddde+j,de-dddde+j,d e-d!dd"de+j,d#e-ddd$de+j,d%e.d&d'd(de+j,d)e.d*d+d,de+j,d-e-d.d/d0de+j,d1e-d2dd3de+j,d4e/dd5d6e+j,d7e0ej1ej2d8d9e+j,d:e/ej2d;d9e+j,de/ej2d?d9e+j,d@e/ej2dAd9e+j,dBe-ej2dCd9e%e+j3dS)DN) DataLoader RandomSampler TensorDataset)AdamWBertForSequenceClassification BertTokenizer@zLoading BERT tokenizer...zbert-base-uncasedT)Z do_lower_casecCs0tj|ddj}|j}tj||kt|S)N)Zaxis)npZargmaxZflattensumlen)ZpredslabelsZ pred_flatZ labels_flatr?/home/ubuntu/amazon-sagemaker-bert-pytorch/code/train_deploy.py flat_accuracysrcCstjdtjtjj|d}|jj}|j j}g}x$|D]}t j |dd}|j |q8Wg} x2|D]*} xt | tkr| j dqhW| j | qbW| }g} x$|D]}dd|D} | j | qWtj|} tj|}tj| }t| ||}|rtjjjj|}nt|}t|||d}|S) NzGet train data loaderz train.csvT)add_special_tokensrcSsg|]}t|dkqS)r)int).0token_idrrr 8sz*_get_train_data_loader..)sampler batch_size)loggerinfopdread_csvospathjoinsentencevalueslabel tokenizerencodeappendr MAX_LENtorchtensorrutilsdata distributedZDistributedSamplerrr)r training_diris_distributeddataset sentencesr input_idssent encoded_sentinput_ids_paddediattention_masksatt_mask train_inputs train_labels train_masks train_data train_samplertrain_dataloaderrrr_get_train_data_loader s6        r<cCstjtjj|d}|jj}|jj}g}x$|D]}tj |dd}|j |q.Wg}x2|D]*} xt | t krv| j dq^W|j | qXW|}g} x$|D]}dd|D} | j | qWt j|} t j|} t j| }t| || }t|}t|||d}|S)Nztest.csvT)rrcSsg|]}t|dkqS)r)r)rrrrrr`sz)_get_test_data_loader..)rr)rrrrrrr r!r"r#r$r r%r&r'rrr)test_batch_sizer+r-r.r r/r0r1r2r3r4r5r6r7r8r9r:r;rrr_get_test_data_loaderJs0       r>c Cs t|jdko|jdk }tjd||jdk}tjd|jtj|rHdnd}|rt|j}t|t j d<|jj |j }t|t j d<t j|j||d tjd |jt jt j|jtj|j|rtjj|jt|j|j|}t|j|j}tjd jt|jt|jd t|jt|jtjd jt|jt|jd t|jt|jtjdtjd|j ddd}|j!|}|r|rtj"j#j$|}n tj"j%|}t&|j'ddd} tjdx$t(d|j)dD]} d} |j*xt+|D]\} } | dj!|}| dj!|}| dj!|}|j,||d||d}|d}| |j-7} |j.tj"j/j0|j'd| j1| |j2dkrtjdj| | t| dt|jd | t||j-qWtjd| t|t|||qWtjdt3|dr |j4n|}|j5|j6ddS)Nr zDistributed training - %srzNumber of gpus available - %dcudacpuZ WORLD_SIZEZRANK)backendZrank world_sizeznInitialized the distributed environment: '%s' backend on %d nodes. Current host rank is %d. Number of gpus: %dz'Processes {}/{} ({:.0f}%) of train datagY@z&Processes {}/{} ({:.0f}%) of test dataz'Starting BertForSequenceClassification zbert-base-uncasedF) num_labelsZoutput_attentionsZoutput_hidden_statesgh㈵>g:0yE>)ZlrZepsz.End of defining BertForSequenceClassification )token_type_idsattention_maskr g?z.Train Epoch: {} [{}/{} ({:.0f}%)] Loss: {:.6f}zAverage training loss: %f zSaving tuned model.module)Zsave_directory)7r ZhostsrArdebugZnum_gpusr&devicestrrenvironindexZ current_hostdistZinit_process_grouprZget_world_sizeZget_rankZ manual_seedZseedr?r<rZdata_dirr>r=testformatrr-rfrom_pretrainedrCtoZnnZparallelZDistributedDataParallelZ DataParallelrZ parametersrangeZepochstrain enumerateZ zero_graditemZbackwardr(Zclip_grad_norm_stepZ log_intervalhasattrrGZsave_pretrained model_dir)argsr,Zuse_cudarIrBZ host_rankZ train_loader test_loadermodelZ optimizerZepochZ total_lossrVbatch b_input_ids b_input_maskb_labelsoutputsZlossZ model_2_saverrrrSos            rSc Cs|jd\}}tjx||D]t}|dj|}|dj|}|dj|}||d|d} | d} | jjj} |jdj} t| | } || 7}q WWdQRXtj d| dS)Nrr rD)rErFr@zTest set: Accuracy: %f )rr) evalr&no_gradrQdetachr@numpyrrr) r[rZrI_Z eval_accuracyr\r]r^r_r`ZlogitsZ label_idsZtmp_eval_accuracyrrrrNs   rNcCsJtjtjjrdnd}tdttj|tj|}td|j |S)Nr?r@z9================ objects in model_dir ===================z9================ model loaded ===========================) r&rIr? is_availableprintrlistdirrrPrQ)rXrIr[rrrmodel_fns  ric CsD|dkr2tj|}tdt|t|tr6|g}n4t|tr\t|dkr\t|dtr\ntdj|t |dd}tdt||d|d }}t j t|t t j t|t }}xRt t||D]@\}\} } t j| ||d t| f<t j| ||d t| f<qWtd t|d ||j|jfStd j|d S)z'An input_fn that loads a pickled tensorzapplication/jsonz0================ input sentences ===============rznUnsupported input type. Input type can be a string or an non-empty list. I got {}T)rzD================ encoded sentences and attension mask ==============r/rFNzB================= padded input and attention mask ================ zUnsupported content type: {})jsonloadsrg isinstancerJlistr ValueErrorrOr"r&Zzerosr%rTzipr'Zlong) request_bodyZrequest_content_typer)Zencodedr/Z attentionZpaddedmaskr3pmrrrinput_fns,   $  " ruc Cstjtjjrdnd}|j||j|\}}|j|}|j|}tdt||tj&|||dd}tdt|WdQRX|S)Nr?r@z-============== encoded data =================)rFrz2=============== inference result =================)r&rIr?rfrQrargrb)Z input_datar[rIZinput_idZ input_maskyrrr predict_fns     rw__main__z --num_labelsrDNz+input batch size for training (default: 64))typedefaultZmetavarhelpz --batch-sizez--test-batch-sizeiz,input batch size for testing (default: 1000)z--epochsz'number of epochs to train (default: 10)z--lrg{Gz?ZLRzlearning rate (default: 0.01)z --momentumg?MzSGD momentum (default: 0.5)z--seedr Szrandom seed (default: 1)z--log-interval2z7how many batches to wait before logging training statusz --backendzIbackend for distributed training (tcp, gloo on cpu and gloo, nccl on gpu))rzr{r|z--hostsZSM_HOSTS)rzr{z--current-hostZSM_CURRENT_HOSTz --model-dirZ SM_MODEL_DIRz --data-dirZSM_CHANNEL_TRAININGz--testZSM_CHANNEL_TESTINGz --num-gpusZ SM_NUM_GPUS)4Zargparserkloggingrsysrdr Zpandasrr&Ztorch.distributedr*rMZtorch.utils.dataZtorch.utils.data.distributedrrrZ transformersrrrZ getLogger__name__rZsetLevelZDEBUGZ addHandlerZ StreamHandlerstdoutr%rgrPr"rr<r>rSrNrirurwZArgumentParserZparserZ add_argumentrfloatrJrnrlrKZ parse_argsrrrrsp   *%h "