3 NbZ@sddlZddlmZmZddlZddlZddlZddlZddl j Z ddl j j Z ddlmZmZddlmZddlmZmZmZddlmZddlmZdd lmZejeZd d Z Gd d d e j!Z"Gddde"eZ#Gddde#Z$Gddde#Z%ddZ&d-ddZ'Gddde"Z(Gddde"Z)ddZ*dd Z+Gd!d"d"e)Z,Gd#d$d$e,Z-Gd%d&d&e,Z.Gd'd(d(e,Z/d.d)d*Z0Gd+d,d,e,Z1dS)/N)abstractmethodABC)Paralleldelayed)linalg)SegmentationModule NUM_CLASS segm_options) InceptionV3)PerceptualLoss)SSIMc Csbtj|dd\}}tj|}t}d}x6t||D](\}}||}|||} | ||<|}q2W|S)z :param groups: group numbers for respective elements :return: dict of kind {group_idx: indices of the corresponding group elements} T) return_countsr)npuniqueargsortdictzip) groupsZ label_groupsZ count_groupsindicesgroupingZ cur_startlabelcountcur_endZ cur_indicesrK/home/ec2-user/SageMaker/lama/saicinpainting/evaluation/losses/base_loss.py get_groupingss  rc@s2eZdZeddZedddZeddZdS) EvaluatorScorecCsdS)Nr)self pred_batch target_batchmaskrrrforward)szEvaluatorScore.forwardNcCsdS)Nr)rrstatesrrr get_value-szEvaluatorScore.get_valuecCsdS)Nr)rrrrreset1szEvaluatorScore.reset)NN)__name__ __module__ __qualname__rr"r$r%rrrrr(s  rcs.eZdZfddZdddZddZZS) PairwiseScorecstjd|_dS)N)super__init__individual_values)r) __class__rrr+7s zPairwiseScore.__init__Nc Cs|dk r$tj|ddjdjjn|j}|j|jd}|dkrL|dfSt}t |}x2|j D]&\}}||} | j| jd||<qdW||fS)a :param groups: :return: total_results: dict of kind {'mean': score mean, 'std': score std} group_results: None, if groups is None; else dict {group_idx: {'mean': score mean among group, 'std': score std among group}} Nr )dim)meanstdr1) torchcatreshapecpunumpyr,r/r0rritems) rrr#r, total_results group_resultsrrindex group_scoresrrrr$;s$ zPairwiseScore.get_valuecCs g|_dS)N)r,)rrrrr%XszPairwiseScore.reset)NN)r&r'r(r+r$r% __classcell__rr)r-rr)6s  r)cs(eZdZdfdd ZdddZZS) SSIMScore cs(tjt|ddj|_|jdS)NF) window_size size_average)r*r+r evalscorer%)rr?)r-rrr+]s zSSIMScore.__init__NcCs.|j||}tj|j|jjjg|_|S)N)rBrhstackr,detachr5r6)rrr r! batch_valuesrrrr"bs zSSIMScore.forward)r>)N)r&r'r(r+r"r<rr)r-rr=\sr=cs(eZdZd fdd Zd ddZZS) LPIPSScorenet-linvggNTcs.tjt||||ddj|_|jdS)NF)modelnet model_pathuse_gpuspatial)r*r+r rArBr%)rrIrJrKrL)r-rrr+ks zLPIPSScore.__init__cCs2|j||j}tj|j|jjjg|_|S)N)rBflattenrrCr,rDr5r6)rrr r!rErrrr"qszLPIPSScore.forward)rGrHNT)N)r&r'r(r+r"r<rr)r-rrFjsrFcCs$tj|dd}tj|dd}||fS)Nr)axisF)rowvar)rr/cov)Zactmusigmarrr#fid_calculate_activation_statisticsysrTư>cCst|\}}t|\}}||}tj|j|dd\}} tj|jsd|} tj| tj |j d|} tj|| j|| }tj |rtj tj |jdddstjtj|j} tdj| |j}tj|} |j|tj|tj|d| S) NF)dispzQfid calculation produces singular product; adding %s to diagonal of cov estimatesrg{Gz?)atolzImaginary component {})rTrZsqrtmdotrisfiniteallLOGGERwarningeyeshape iscomplexobjallclosediagonalimagmaxabs ValueErrorformatrealtrace)activations_predactivations_targetepsZmu1Zsigma1Zmu2Zsigma2diffZcovmean_msgoffsetmZ tr_covmeanrrrcalculate_frechet_distances"     rrcsBeZdZdfdd ZdddZddd Zd d Zd d ZZS)FIDScoreư>csbtjdtjttdddkr>tj|}t|gjt_ tj |_ ||_ |j tjddS)NzFIDscore init called_MODELzFIDscore init done) r\infor*r+getattrrsr BLOCK_INDEX_BY_DIMrArvrIrlr%)rdimsrl block_idx)r-rrr+s   zFIDScore.__init__NcCsD|j|}|j|}|jj|jj|jj|jj||fS)N)_get_activationsrjappendrDr5rk)rrr r!rjrkrrrr"s   zFIDScore.forwardc Cstjd|dk rt|n |j|jf\}}tj|jj}tj|jj}t |||j d}t |d}|dkrvd}njt }t |}xZ|j D]N\} } t| dkrt || || |j d} t | d|| <qt tdd|| <qW|jtjd||fS)NzFIDscore get_value called)rl)r/r nanzFIDscore get_value done)r\rwrrjrkr2r3r5r6rrrlrrr7lenfloatr%) rrr#rjrkZtotal_distancer8r9rrr:Zgroup_distancerrrr$s(     zFIDScore.get_valuecCsg|_g|_dS)N)rjrk)rrrrr%szFIDScore.resetcCsJ|j|d}|jddks*|jddkr6ds6td|jdjd}|S) NrrXr FzNWe should not have got here, because Inception always scales inputs to 299x299r1r1)rIr_AssertionErrorsqueeze)rbatch activationsrrrr|s zFIDScore._get_activations)rtru)N)NN) r&r'r(r+r"r$r%r|r<rr)r-rrss  rscs0eZdZfddZddZfddZZS)SegmentationAwareScorecs2tjt|ddj|_g|_g|_g|_dS)NT) weights_pathZuse_default_normalization)r*r+rrA segm_network target_class_freq_by_image_totaltarget_class_freq_by_image_maskpred_class_freq_by_image_mask)rr)r-rrr+s  zSegmentationAwareScore.__init__cCsZ|jj|dj|jddjjjj}|jj|dj|jddjjjj}|j|jdd dkjjj}g}g}g} xt|||D]\} } } t j | t dd } t j | | t dd }t j | | t dd }|j j | |jj ||jj ||j | |j || j |qWt j|dd}t j|dd}t j| dd} ||| fS) Nrr g?) minlength.)rOr1r1r1)N.)N.)N.)rpredictviewr_longrDr5r6rrbincountrrr}rr concatenate)rrr r!Zpred_segm_flatZtarget_segm_flatZ mask_flatZbatch_target_class_freq_totalZbatch_target_class_freq_maskZbatch_pred_class_freq_maskZ cur_pred_segmZcur_target_segmZcur_maskZcur_target_class_freq_totalZcur_target_class_freq_maskZcur_pred_class_freq_maskrrrr"s(.."     zSegmentationAwareScore.forwardcs tjg|_g|_g|_dS)N)r*r%rrr)r)r-rrr%s zSegmentationAwareScore.reset)r&r'r(r+r"r%r<rr)r-rrs rcsb|jdkr|jd|jdks"t|jd||djd}|d}fddt|DS)NrXr.gMbP?cs&i|]\}}|dkr||qS)rr).0ival)idx2nametotal_class_freqrr sz0distribute_values_to_classes..).N)ndimr_rsum enumerate)rvaluesrZ distr_valuesresultr)rrrdistribute_values_to_classess "  rcCs(ddtdjddddjjDS)NcSsi|]\}}||dqS)r r)rrnamerrrrsz-get_segmentation_idx2name..classesZIdxT)dropName)r Z set_indexto_dictr7rrrrget_segmentation_idx2namesrcsJeZdZfddZfddZeddZd dd Zfd d ZZ S) SegmentationAwarePairwiseScorecs tj||g|_t|_dS)N)r*r+r,r segm_idx2name)rargskwargs)r-rrr+sz'SegmentationAwarePairwiseScore.__init__cs4tj|||}|j|||}|jj|||fS)N)r*r" calc_scorer,r})rrr r!Zcur_class_statsZ score_values)r-rrr"s z&SegmentationAwarePairwiseScore.forwardcCs tdS)N)NotImplementedError)rrr r!rrrrsz)SegmentationAwarePairwiseScore.calc_scoreNcCs|dk r|\}}}}n|j}|j}|j}|j}tj|dd}tj|dd}tj|dd}tj|dd}|j|jdt|||j }|dkr|dfSt }t |} xH| j D]<\} } || } || } | j| jdt| | |j || <qW||fS)a :param groups: :return: total_results: dict of kind {'mean': score mean, 'std': score std} group_results: None, if groups is None; else dict {group_idx: {'mean': score mean among group, 'std': score std among group}} Nr)rO)r/r0) rrrr,rrr/r0rrrrr7)rrr#rrrr,r8r9rrr:group_class_freqr;rrrr$#s0  z(SegmentationAwarePairwiseScore.get_valuecstjg|_dS)N)r*r%r,)r)r-rrr%Ps z$SegmentationAwarePairwiseScore.reset)NN) r&r'r(r+r"rrr$r%r<rr)r-rrs     -rc@seZdZddZdddZdS)SegmentationClassStatscCsdS)Nrr)rrr r!rrrrVsz!SegmentationClassStats.calc_scoreNcs|dk r|\}}}}nj}j}j}tj|dd}tj|dd}tj|dd}|jdjdj|jdjd}||j}||jd|jdd}t} | jfddt D| jfddt |D| jfd dt |D|dkr| dfSt} t |} x| j D]\} } || }|| }|| }|jdjdj|jdjd}||j}||jd|jdd}t}|jfd dt D|jfd dt |D|jfd dt |D|| | <q,W| | fS) a :param groups: :return: total_results: dict of kind {'mean': score mean, 'std': score std} group_results: None, if groups is None; else dict {group_idx: {'mean': score mean among group, 'std': score std among group}} Nr)rOfloat32gMbP?cs*i|]"\}}|dkr|dj|qS)rz total_freq/)r)rrv)rrrrxsz4SegmentationClassStats.get_value..cs*i|]"\}}|dkr|dj|qS)rz mask_freq/)r)rrr)rrrr{scs.i|]&\}}|dkr|dj|qS)rzmask_freq_diff/)r)rrr)r)target_class_freq_by_image_total_marginalrrr~scs*i|]"\}}|dkr|dj|qS)rz total_freq/)r)rrr)rrrrscs*i|]"\}}|dkr|dj|qS)rz mask_freq/)r)rrr)rrrrscs.i|]&\}}|dkr|dj|qS)rzmask_freq_diff/)r)rrr)/group_target_class_freq_by_image_total_marginalrrrrs) rrrrrrastyperupdaterrr7)rrr#rrrrnZ(target_class_freq_by_image_mask_marginalZpred_class_freq_diffr8r9rrr:Z&group_target_class_freq_by_image_totalZ%group_target_class_freq_by_image_maskZ#group_pred_class_freq_by_image_maskZ.group_target_class_freq_by_image_mask_marginalZgroup_pred_class_freq_diffZcur_group_resultsr)rrrrr$YsV      z SegmentationClassStats.get_value)NN)r&r'r(rr$rrrrrUsrcs*eZdZddfdd ZddZZS)SegmentationAwareSSIMr>)r?cs$tj||t|ddj|_dS)NF)r?r@)r*r+r rA score_impl)rr?rr)r-rrr+szSegmentationAwareSSIM.__init__cCs|j||jjjS)N)rrDr5r6)rrr r!rrrrsz SegmentationAwareSSIM.calc_score)r&r'r(r+rr<rr)r-rrsrcs0eZdZdddddfdd Zdd ZZS) SegmentationAwareLPIPSznet-linrHNT)rIrJrKrLcs*tj||t||||ddj|_dS)NF)rIrJrKrLrM)r*r+r rAr)rrIrJrKrLrr)r-rrr+szSegmentationAwareLPIPS.__init__cCs|j||jjjjS)N)rrNrDr5r6)rrr r!rrrrsz!SegmentationAwareLPIPS.calc_score)r&r'r(r+rr<rr)r-rrsrcCs"|j}||||<t|||dS)N)rl)copyrr)img_irjrkrlrrrcalculade_fid_no_imgs rcsHeZdZddddfdd ZddZdd d Zd d ZddZZS)SegmentationAwareFIDigư>r )rzrln_jobscsPtj||ttdddkr8tj|}t|gjt_tj|_||_ ||_ dS)Nrv) r*r+rxrsr ryrArvrIrlr)rrzrlrrrr{)r-rrr+s zSegmentationAwareFID.__init__cCs|j|}|j|}||fS)N)r|)rrr r!rjrkrrrrs  zSegmentationAwareFID.calc_scoreNcCsD|dk r|\}}}}n|j}|j}|j}|j}tj|dd}tj|dd}tj|dd}t|\}}tj|dd}tj|dd}t|||jddd|j |||} |dkr| dfSt } t |} xx| j D]l\} } t | dkr$|| }|| }|| }t|||jddd|j |||| | <qt tddd| | <qW| | fS)a :param groups: :return: total_results: dict of kind {'mean': score mean, 'std': score std} group_results: None, if groups is None; else dict {group_idx: {'mean': score mean among group, 'std': score std among group}} Nr)rO)rl)r/r0r r~)rrrr,rrrrrrldistribute_fid_to_classesrrr7rr)rrr#rrrZactivation_pairsrjrkr8r9rrr:Zgroup_activations_predZgroup_activations_targetrrrrr$s> zSegmentationAwareFID.get_valuecsRtjd}tjdfddtjdD}||}t||jS)N)rl)rc3s$|]}tt|jdVqdS))rlN)rrrl)rr)rjrkrrr szASegmentationAwareFID.distribute_fid_to_classes..r)rrrlrrranger_rr)rZ class_freqrjrkZreal_fidZ fid_no_imageserrorsr)rjrkrrrs  z.SegmentationAwareFID.distribute_fid_to_classescCsX|j|d}|jddks*|jddkr8tj|dd}|jdjdjjj}|S) NrrXr r) output_size)r r r1r1)rIr_Fadaptive_avg_pool2drrDr5r6)rrrrrrr| s z%SegmentationAwareFID._get_activationsr1)NN) r&r'r(r+rr$rr|r<rr)r-rrs   5 r)rU)rU)2loggingabcrrr6rZsklearnZ sklearn.svmr2torch.nnnnZtorch.nn.functional functionalrjoblibrrscipyrZ models.ade20krrr Z fid.inceptionr lpipsr ssimr getLoggerr&r\rModulerr)r=rFrTrrrsrrrrrrrrrrrrrs>       & A*BO