U ~d@sdZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlZddlZddlmZddlmZddlmZmZddlmZddlmmZddl Z ddlZddl mZe d d d Z!d d Z"ddZ#ddZ$Gddde%Z&ddZ'Gddde%Z(Gddde%Z)Gddde%Z*Gddde%Z+Gddde%Z,Gd d!d!e%Z-d"d#Z.dtd$d%Z/d&d'Z0dud)d*Z1dvd,d-Z2dwd/d0Z3d1d2Z4d3d4Z5d5d6Z6d7d8Z7dxd9d:Z8dyd;d<Z9d=d>Z:dzd@dAZ;GdBdCdCe%ZdIdJZ?dKdLZ@dMdNZAdOdPZBdQdRZCdSdTZDdUdVZEdWdXZFd|dZd[ZGd\d]ZHd}dbdcZIGdddedeejJjKZLGdfdgdgejMZNdhdiZOdjdkZPGdldmdmZQdndoZRgfdpdqZSdrdsZTdS)~z Misc functions. Mostly copy-paste from torchvision references or other public repos like DETR: https://github.com/facebookresearch/detr/blob/master/util/misc.py )nan)utilsN) defaultdictdeque)nn) ImageFilterImageOps) timedeltaignorecCs"ttdd|}tjjj|S)NcSs|dk SNxr r g/apps/aws-distributed-training-workshop-pcluster/head-node-scripts/without_snakemake/pyscripts/utils.py.zcollate_fn..)listfiltertorchrdata dataloaderdefault_collate)batchr r r collate_fn-srcCst|rdSdSdSNTF)npisnananyr r r r check_nan2srcCs4t|dk}t|ddkr0td|dSdS)Nrzx_0: {}T)rwherelenprintformat)rx_0r r r check_zero9sr$cCs|jddd}|jddd}||}t|r4tdt||rHtdt|rXtdt|rtdtd|td |td |td |j||||}t|rtd |S) N)Taxiskeepdimsdiff_min_max is nanx-x_min is nan:#x contains nan before normalizationdiff_min_max has zerox_maxx_min diff_min_maxr"x contains nan after normalizationminmaxrr!r$shaperr/r.r0r r rnormalize_tensor_per_channel@s&     r7c@seZdZddZdS)normalize_tensor_0_to_1cCs"|}t|}t|}|Sr )numpyr7r from_numpyfloat)selftensorimager r r__call__Wsz normalize_tensor_0_to_1.__call__N)__name__ __module__ __qualname__r?r r r rr8Vsr8cCs|jddd}|jddd}||}t|r4tdt||rHtdt|rXtdt|rtdtd|td |td |td |j||||}t|rtd |S) N)rr%Tr'r*r+r,r-r.r/r0rr1r2r6r r rnormalize_numpy_0_to_1^s&     rCc@s"eZdZdZd ddZddZd S) GaussianBlurz/ Apply Gaussian Blur to the PIL image. ?皙?@cCs||_||_||_dSr )prob radius_min radius_max)r<prIrJr r r__init__xszGaussianBlur.__init__cCs4t|jk}|s|S|tjt|j|jdS)N)radius)randomrHrrrDuniformrIrJ)r<imgdo_itr r rr?}szGaussianBlur.__call__N)rErFrGr@rArB__doc__rLr?r r r rrDts rDc@s"eZdZdZd ddZddZd S) "GaussianBlur_forGreyscaleMultiChanz* Apply Gaussian Blur to a Tensor. rErFrGcCs||_||_||_dSr )rH sigma_min sigma_max)r<rKrUrVr r rrLsz+GaussianBlur_forGreyscaleMultiChan.__init__cCs4t|jk}|s|Stjddd}||}|SdS)N)rW)rFg?) kernel_sizesigma)rNrH transformsrD)r<r=rQblurr r rr?s z+GaussianBlur_forGreyscaleMultiChan.__call__N)rErFrGrRr r r rrTs rTc@s eZdZdZddZddZdS) Solarizationz. Apply Solarization to the PIL image. cCs ||_dSr rKr<rKr r rrLszSolarization.__init__cCs t|jkrt|S|SdSr )rNrKrsolarize)r<rPr r rr?s zSolarization.__call__NrRr r r rr\sr\c@s"eZdZdZdddZddZdS) "Solarization_forGreyscaleMultiChanz+ Apply Solarization to the Tensor. rEcCs||_||_dSr )rK threshold)r<rKrar r rrLsz+Solarization_forGreyscaleMultiChan.__init__cCs6t|jkr.d|||jk|||jk<|S|SdSNr%)rNrKra)r<r=r r rr?sz+Solarization_forGreyscaleMultiChan.__call__N)rErRr r r rr`s r`c@seZdZddZddZdS)AdjustBrightnesscCs ||_dSr r]r^r r rrLszAdjustBrightness.__init__cCsJt|jkrBt|jdD] }t||tdd||<q|S|SdS)NrrEg?)rNrKranger5TFadjust_brightnessrO)r<rchannelr r rr?s zAdjustBrightness.__call__Nr@rArBrLr?r r r rrcsrcc@seZdZdddZddZdS)AdjustGamma_custom:0yE>cCs||_||_dSr )rKeps)r<rKrkr r rrLszAdjustGamma_custom.__init__cCsRt|jkrJtdd}t|jdD]}t||||jd||<q(|S|SdS)Ng?r)rk)rNrKrOrdr5gamma_adjust_wo_nanrk)r<rZ random_gammargr r rr?s  zAdjustGamma_custom.__call__N)rjrhr r r rris ricCs$|j|d}|d}|d}|S)Nr/imgssplit)datasetindeximg_pathZimg_folder_namer r r&fetch_foldername_of_img_location_donors rvcCs|j|dd| dSNrrnr&rp)rsrt folder_depthr r r fetch_foldername_of_img_locationsrycCsddt|DS)NcSsg|]}|r|jqSr )is_dirname).0fr r r sz-list_folders_in_directory..)osscandir) directoryr r rlist_folders_in_directorysrr%cCs8|j|d}d|dd| d}t|}|Srw)rqjoinrrr)rsrtrxpath class_namesr r r&fetch_all_folder_names_of_folder_depthsrrjcCs:|}t||||t|}t|}|Sr )r9rabssignrr:r;)r=gammarkargZgamma_adj_channelZ tensor_adjr r rrms rmteacherc Cstj|rtj|dd}|dk rR||krRtd|d||}|djd}nfz|djd}WnRtk r}z4t|td|td td t dW5d}~XYnX|SdS) Ncpu map_location Take key  in provided checkpoint dictz backbone.patch_embed.proj.weightr%zpatch_embed.proj.weightz&in_chans not found in checkpoint dict.zs z+load_pretrained_weights..cSsi|]\}}|dd|qS)z backbone.rrrr r rr s Fstrictz6Pretrained weights found at {} and loaded with msg: {}zbPlease use the `--pretrained_weights` argument to indicate the path of the checkpoint to evaluate. vit_smallz7dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pthz5dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pthvit_basez3dino_vitbase16_pretrain/dino_vitbase16_pretrain.pthz1dino_vitbase8_pretrain/dino_vitbase8_pretrain.pthZxcit_small_12_p16zCdino_xcit_small_12_p16_pretrain/dino_xcit_small_12_p16_pretrain.pthZxcit_small_12_p8zAdino_xcit_small_12_p8_pretrain/dino_xcit_small_12_p8_pretrain.pthZxcit_medium_24_p16zEdino_xcit_medium_24_p16_pretrain/dino_xcit_medium_24_p16_pretrain.pthZxcit_medium_24_p8zCdino_xcit_medium_24_p8_pretrain/dino_xcit_medium_24_p8_pretrain.pthresnet50z1dino_resnet50_pretrain/dino_resnet50_pretrain.pthzPThere is no reference weights available for this model => We use random weights.) rrrrrr!itemsload_state_dictr")modelrr model_name patch_sizermsgurlr r rload_pretrained_weightss<     rcCsd}|dkr|dkrd}nN|dkr0|dkr0d}n8|dkrF|dkrFd}n"|dkr\|dkr\d}n |d krhd }|dk rtd tjjd |d d}|j|ddntddS)Nrrzr%)named_parametersgradrnormappenditemmul_)rclipnormsr{rK param_norm clip_coefr r rclip_gradients:s  rcCs0||kr dS|D]\}}d|krd|_qdS)NZ last_layer)rr)epochrZfreeze_last_layernrKr r rcancel_gradients_last_layerFs rc Kstj|sdStd|tj|dd}|D]\}}||kr|dk rz(|j||dd}td|||Wqt k rz"|||}td||Wn$t k rtd ||YnXYqXq4td ||q4|dk r|D]}||kr||||<qdS) z" Re-start from checkpoint NzFound checkpoint at {}rrFrz/=> loaded '{}' from checkpoint '{}' with msg {}z$=> loaded '{}' from checkpoint: '{}'z,=> failed to load '{}' from checkpoint: '{}'z)=> key '{}' not found in checkpoint: '{}') rrrr!r"rrrr TypeError ValueError)Zckp_pathZ run_variableskwargs checkpointkeyvaluervar_namer r rrestart_from_checkpointNs(  rc Cstg}||}|dkr(t|||}t|||}|d||dttj|t|} t|| f} t| ||kst| S)NrrEr%) rarraylinspacearangecospir concatenateAssertionError) Z base_valueZ final_valueepochsZ niter_per_epZ warmup_epochsZstart_warmup_valueZwarmup_scheduleZ warmup_itersitersscheduler r rcosine_schedulerqs ,rcCsBdddh}dddh}||kr$dS||kr4dStd d S) z8 Parse boolean arguments from the command line. offfalse0ontrue1FTz invalid value for a boolean flagN)lowerargparseZArgumentTypeError)sZ FALSY_STRINGSZTRUTHY_STRINGSr r r bool_flags    rcCs&t|tj|tj|dS)z Fix random seeds. N)r manual_seedcudamanual_seed_allrrNseed)rr r rfix_random_seedss  rc@speZdZdZdddZdddZd d Zed d Zed dZ eddZ eddZ eddZ ddZ dS) SmoothedValuezsTrack a series of values and provide access to smoothed values over a window or the global series average. NcCs.|dkr d}t|d|_d|_d|_||_dS)Nz{median:.6f} ({global_avg:.6f}))maxlenr)rtotalcountfmt)r< window_sizerr r rrLs  zSmoothedValue.__init__r%cCs0|j||j|7_|j||7_dSr )rrrr)r<rrr r rupdates zSmoothedValue.updatecCsZts dStj|j|jgtjdd}tt|| }t |d|_|d|_dS)z: Warning: does not synchronize the deque! Nr)dtypedevicerr%) is_dist_avail_and_initializedrr=rrfloat64distbarrier all_reducetolistint)r<tr r rsynchronize_between_processess z+SmoothedValue.synchronize_between_processescCstt|j}|Sr )rr=rrmedianrr<dr r rrszSmoothedValue.mediancCs"tjt|jtjd}|S)N)r)rr=rrfloat32meanrrr r ravgszSmoothedValue.avgcCs |j|jSr )rrr<r r r global_avgszSmoothedValue.global_avgcCs t|jSr )r4rrr r rr4szSmoothedValue.maxcCs |jdS)N)rrr r rrszSmoothedValue.valuecCs |jj|j|j|j|j|jdS)N)rrrr4r)rr"rrrr4rrr r r__str__szSmoothedValue.__str__)rN)r%)r@rArBrSrLrrpropertyrrrr4rrr r r rrs        rTc Cst}|dkr|Stpg}g}t|D]}|||||q0tj|dd}t||rr||}ddt ||D}W5QRX|S)a6 Args: input_dict (dict): all the values will be reduced average (bool): whether to do average or sum Reduce the values in the dictionary from all processes so that all processes have the averaged results. Returns a dict with the same fields as input_dict, after reduction. r&r)dimcSsi|]\}}||qSr r rr r rrszreduce_dict..) get_world_sizerno_gradsortedrrstackrrzip)Z input_dictaverage world_sizenamesvaluesrZ reduced_dictr r r reduce_dicts    rc@sHeZdZdddZddZddZdd Zd d Zd d ZdddZ dS) MetricLogger cCstt|_||_dSr )rrmeters delimiter)r<rr r rrLs zMetricLogger.__init__cKsL|D]>\}}t|tjr$|}t|ttfs6t|j| |qdSr ) r isinstancerTensorrr;rrrr)r<rrrr r rrs  zMetricLogger.updatecCsB||jkr|j|S||jkr(|j|Stdt|j|dS)Nz!'{}' object has no attribute '{}')r__dict__AttributeErrorr"typer@)r<attrr r r __getattr__s    zMetricLogger.__getattr__cCs:g}|jD]\}}|d|t|q|j|S)Nz{}: {})rrrr"strrr)r<Zloss_strr{meterr r rrs zMetricLogger.__str__cCs|jD] }|q dSr )rrr)r<rr r rr sz*MetricLogger.synchronize_between_processescCs||j|<dSr )r)r<r{rr r r add_meterszMetricLogger.add_meterNc csd}|s d}t}t}tdd}tdd}dtttt|d} tjrz|j|d| dd d d d d g} n |j|d| dd d d d g} d} |D]} | t|| V| t|||dks|t|dkr|j t||} tt j t | d}tjrXt| j|t||t|t|t|tj| dn*t| j|t||t|t|t|d|d7}t}qt|}tt j t |d}td|||t|dS)Nrrz {avg:.6f})r:rz[{0z}/{1}]z eta: {eta}z{meters}z time: {time}z data: {data}zmax mem: {memory:.0f}g0Ar%seconds)etartimermemory)r!rr"rz!{} Total time: {} ({:.6f} s / it))r"rrr rr is_availablerrrrdatetimer rr!r"max_memory_allocated)r<iterableZ print_freqheaderi start_timeendZ iter_timeZ data_timeZ space_fmtZlog_msgMBobjZ eta_secondsZ eta_string total_timeZtotal_time_strr r r log_everysz           zMetricLogger.log_every)r)N) r@rArBrLrrrrrr/r r r rrs rcstjtjtfdd}d}d}d}zN|dddg}tjddgd |dd dg}|rdd nd}|ddd dg}Wntk rYnXd |d|d|}|S)Ncstj|ddS)Ncwdascii) subprocess check_outputdecodestrip)Zcommandr0r r_runLszget_sha.._runzN/Acleangitz rev-parseHEADdiffr0z diff-indexzhas uncommited changesz --abbrev-refzsha: z , status: z , branch: )rrdirnameabspath__file__r3r4r)r7shar;branchmessager r0rget_shaIs  rBcCsts dStsdSdS)NFT)rr$is_initializedr r r rr]s rcCsts dStSrb)rrrr r r rresrcCsts dStSNr)rrget_rankr r r rrEksrEcCs tdkSrD)rEr r r ris_main_processqsrFcOstrtj||dSr )rFrsave)argsrr r rsave_on_masterusrIcs&ddl}|jfdd}||_dS)zD This function disables printing when not in master process rNcs"|dd}s|r||dS)NforceF)pop)rHrrJZ builtin_print is_masterr rr!s z$setup_for_distributed..print)builtinsr!)rM __builtin__r!r rLrsetup_for_distributedzsrPcCsLdtjkrFdtjkrFttjd|_ttjd|_ttjd|_ndtjkrtttjd|_|jtj|_nxtj rt dd\|_|_|_d t dd tjd <tt d d tjd <t d tjd nt dtdtjd|j|j|jtdddtj|jt d |j|jddtt|jdkdS)NRANK WORLD_SIZE LOCAL_RANKZ SLURM_PROCIDzWill run the code on one GPU.)rrr%z 127.0.0.{}r% MASTER_ADDRi   rdr%csZt|}|d||ddd\}}|}||dd|fdd|DS)zNComputes the accuracy over the k top predictions for the specified values of krr%Trcs0g|](}d|dddqS)NrrY@)reshaper;sum)r|r batch_sizeZcorrectr rr~szaccuracy..)r4sizetopkreqrg expand_as)outputtargetrlZmaxk_predr riraccuracys  rsc Csdd}||d|ks(||d|kr6tjdddt||||}||||}|d|dd|d|||td| ||j ||d|W5QRSQRXdS) NcSsdt|tddS)N?rG)matherfsqrtr r r rnorm_cdfsz(_no_grad_trunc_normal_..norm_cdfr&zjmean is more than 2 std from [a, b] in nn.init.trunc_normal_. The distribution of values may be incorrect.) stacklevelr%rG)r3r4) warningswarnrruniform_erfinv_rrurwadd_clamp_)r=rstdrbrxlur r r_no_grad_trunc_normal_s   rrrtrGcCst|||||Sr )r)r=rrrrr r r trunc_normal_srcs2eZdZdZd fdd Zedd ZZS) LARSzb Almost copy-paste from https://github.com/facebookresearch/barlowtwins/blob/main/main.py r?MbP?Nc s&t||||||d}t||dS)N)lr weight_decaymomentumr!weight_decay_filterlars_adaptation_filter)dictsuperrL) r<paramsrrrr!rrdefaults __class__r rrLs z LARS.__init__c Cs|jD]}|dD]}|j}|dkr&q|jdkrB|j||dd}|jdkrt|}t|}t|}t|dkt|dk|d||||}||}|j |}d|krt ||d<|d} | |d  ||j | |d  dqqdS) Nrr%r)alpharrr!murr) param_groupsrndimaddrr ones_likermulstate zeros_likerr~) r<grKdprZ update_normoneqZ param_staterr r rsteps2           z LARS.step)rrrrNN) r@rArBrSrLrrr __classcell__r r rrrsrcs(eZdZdZfddZddZZS)MultiCropWrappera| Perform forward pass separately on each resolution input. The inputs corresponding to a single resolution are clubbed and single forward is run on the same resolution inputs. Hence we do several forward passes = number of different resolutions used. We then concatenate all the output features and run the head forward on these concatenated features. cs4tt|tt|_|_||_||_dSr )rrrLrIdentityfcheadbackbone)r<rrrr rrLszMultiCropWrapper.__init__cCst|ts|g}ttjtdd|Ddddd}dtd|dj}}|D]@}| t |||}t|t r|d}t ||f}|}qZ| |S)NcSsg|]}|jdqS)r)r5)r|inpr r rr~"sz,MultiCropWrapper.forward..T) return_countsr%r) rrrcumsumunique_consecutiver=emptytorrcattupler)r<rZ idx_crops start_idxroend_idxZ_outr r rforwards$  zMultiCropWrapper.forward)r@rArBrSrLrrr r rrr s rcCsbg}g}|D]>\}}|js q|ds8t|jdkrD||q||qd|i|ddgS)Nz.biasr%rr)rr)r requires_gradendswithr r5r)rZ regularizedZnot_regularizedr{paramr r rget_params_groups3s  rcCs:tjtjtjtjf}|D]\}}t||rdSqdSr)r BatchNorm1d BatchNorm2d BatchNorm3d SyncBatchNorm named_modulesr)rZbn_typesr{moduler r rhas_batchnormsAs  rc@s*eZdZdZd ddZddZdd Zd S) PCAz* Class to compute and apply PCA. rEcCs||_||_d|_dSr )rwhitr)r<rrr r rrLMsz PCA.__init__cCstj|\}}|d}||k}|dkr<||||k<|}t|dddd|j}||}|dd|f}td||dtd||j }t ||j |_ dS)zB Takes a covariance matrix (np.ndarray) as input. gh㈵>rNrzkeeping %.2f %% of the energyrfrt) rlinalgeighr4rhargsortrr!diagrdotTdvt)r<covrrrkZn_0Z totenergyidxr r r train_pcaRs   z PCA.train_pcacCst|tjr2|jdk r ||j8}t|j|jjS|jrz|jdk rT|tj |j8}t tj |j| dd ddS|jdk r|t |j8}t t |j| dd ddS)Nrr%) rrndarrayrrrris_cudarr FloatTensormm transpose)r<rr r rapplyls    & z PCA.applyN)rrE)r@rArBrSrLrrr r r rrIs rc Csrt|}d}d|}t|D]N}||}|dkr8d}n t||}t|d|d}||||d7}q|S)z Computes average precision for given ranked indexes. Arguments --------- ranks : zerro-based ranks of positive images nres : number of positive images Returns ------- ap : average precision rrtr%rG)r rrr;) ranksZnresZ nimgranksapZ recall_stepjr[Z precision_0Z precision_1r r r compute_aps  rcCs&d}t|}t|}tt|}t|t|f}d}t|D]} t|| d} | jddkrtd|| <td|| ddf<|d7}qDzt|| d} Wntd} YnXt|jdt|dd| f| } t|jdt|dd| f| } d}d}t| rd}|t| kr|t| krj| || |krj|d7}|d7}q6| ||| |<|d7}q(t | t| }||}||| <| d7} tt|D]0}t t | ||}| |k ||| |f<q||| ddf}qD|||}|||}||||fS)a Computes the mAP for a given set of returned results. Usage: map = compute_map (ranks, gnd) computes mean average precsion (map) only map, aps, pr, prs = compute_map (ranks, gnd, kappas) computes mean average precision (map), average precision (aps) for each query computes mean precision at kappas (pr), precision at kappas (prs) for each query Notes: 1) ranks starts from 0, ranks.shape = db_size X #queries 2) The junk results (e.g., the query itself) should be declared in the gnd stuct array 3) If there are no positive images for some query, that query is excluded from the evaluation rrokrNr%junk) r rzerosrrr5r;rin1drr3r4rh)rZgndZkappasmapZnqZapsprZprsZnemptyr)ZqgndZqgndjposrrijiprrZkqr r r compute_mapsP  ((      rcCsnd}dD]L}|dkr|}ntjj||ddd}||}|dkrL|}q||7}q|d}||}|S)N)r%g;f?rEr%bilinearF) scale_factormode align_cornersrl)cloner functional interpolater)samplesrrrrZfeatsr r r multi_scales    r)r)r%r)rjr%)r)N)rr)r)T)re)rrtrrG)UrScmathremailrrrr"rurNr%r3 collectionsrrr9rrrtorch.distributed distributedrPILrrtorchvision.transformsrZZ!torchvision.transforms.functionalrrerzr filterwarningsrrr$r7objectr8rCrDrTr\r`rcrirvryrrrmrrrrrrrrrrrrrBrrrErFrIrPrdrsrroptim OptimizerrModulerrrrrrrr r r rs            %  #  > Z( $ )&6# H