ó —Àv]c@@s=dZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z de fd„ƒYZdZd„Zd „Zd e fd „ƒYZd e fd „ƒYZde fd„ƒYZdd„Zddd„Zd„Zd„Zedkr9eƒndS(sº Tracker script for DMLC Implements the tracker control protocol - start dmlc jobs - start ps scheduler and rabit tracker - help nodes to establish links with each other Tianqi Chen i(tabsolute_importN(tThreadtExSocketcB@sDeZdZd„Zd„Zd„Zd„Zd„Zd„ZRS(sE Extension of socket to handle recv and send of special data cC@s ||_dS(N(tsock(tselfR((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt__init__scC@shg}d}xL||krZ|jjt||dƒƒ}|t|ƒ7}|j|ƒqWdj|ƒS(Niit(Rtrecvtmintlentappendtjoin(Rtnbytestrestnreadtchunk((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytrecvallscC@stjd|jdƒƒdS(Ns@iii(tstructtunpackR(R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytrecvint&scC@s |jjtjd|ƒƒdS(Ns@i(RtsendallRtpack(Rtn((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytsendint(scC@s-|jt|ƒƒ|jj|jƒƒdS(N(RR RRtencode(Rts((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytsendstr*scC@s|jƒ}|j|ƒjƒS(N(RRtdecode(Rtslen((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytrecvstr-s ( t__name__t __module__t__doc__RRRRRR(((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyRs     i™ÿcC@stj|dƒdddS(Nii(tsockett getaddrinfotNone(thost((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt get_some_ip4scC@stj|dƒddS(Ni(R!R"R#(taddr((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt get_family7st SlaveEntrycB@s#eZd„Zd„Zd„ZRS(cC@s¸t|ƒ}||_t|dƒ|_|jƒ}|tksYtd||jfƒ‚|jtƒ|jƒ|_|jƒ|_ |j ƒ|_ |j ƒ|_ d|_ d|_dS(Nisinvalid magic number=%d from %s(RRR%R$RtkMagictAssertionErrorRtrankt world_sizeRtjobidtcmdt wait_acceptR#tport(RRts_addrtslavetmagic((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyR;s   %  cC@sC|jdkr|jS|jdkr?|j|kr?||jSdS(NitNULLiÿÿÿÿ(R+R-(Rtjob_map((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt decide_rankIs  cC@s||_t||ƒ}||\}}|jj|ƒ|jj||ƒ|jjt|ƒƒ|jjt|ƒƒx|D]} |jj| ƒq€W|dkrÒ||krÒ|j|ƒ|jj|ƒn|jjdƒ|dkr||kr|j|ƒ|jj|ƒn|jjdƒxätr|jjƒ} tgƒ} x*t| ƒD]} | j|jjƒƒq[W| j |ƒst ‚|| } g}x*| D]"} | |kr§|j | ƒq§q§W|jjt|ƒƒ|jjt| ƒt|ƒƒxL|D]D} |jj || j ƒ|jj|| jƒ|jj| ƒq W|jjƒ}|dkrsq-n|jjƒ|_g}xD|D]<} || jd8_|| jdkr’|j | ƒq’q’Wx|D]} |j| dƒqÙWt| ƒt|ƒ|_|SWdS(Niÿÿÿÿii(R+tsetRRR taddtTrueRtrangetissubsetR*R RR$R0R/tpopR#(RR+t wait_connttree_mapt parent_maptring_maptnnsettrprevtrnexttrtngoodtgoodsett_tbadsettconsettnerrtrmset((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt assign_rankPs^              (RRRR6RL(((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyR(:s  t RabitTrackercB@s†eZdZddd„Zd„Zed„ƒZd„Zd„Zd„Z d „Z d „Z d „Z d „Z d „Zd„ZRS(s tracker for rabit iƒ#i'cC@sätjt|ƒtjƒ}xit||ƒD]X}y!|j||fƒ||_PWq+tjk r‚}|jdkr|q+qƒ‚q+Xq+W|jdƒ||_ ||_ d|_ d|_ d|_||_tjd||jƒdS(Nibi0isstart listen on %s:%d(ibi0(R!R't SOCK_STREAMR:tbindR0terrorterrnotlistenRthostIPR#tthreadt start_timetend_timetnslavetloggingtinfo(RRSRWR0tport_endRte((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyRs$         cC@s|jjƒdS(N(Rtclose(R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt__del__¢scC@sˆ|d}g}|dkr4|j|ddƒn|dd|kr`|j|ddƒn|d|kr„|j|dƒn|S(Nii(R (R+RWtret((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt get_neighbor¥s  cC@si|jd6|jd6S(s^ get enviroment variables for slaves can be passed in as args or envs tDMLC_TRACKER_URItDMLC_TRACKER_PORT(RSR0(R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt slave_envs±s cC@sYi}i}x@t|ƒD]2}|j||ƒ||<|ddd||R?RD((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytget_tree¹s c C@s¬t||ƒ}|t||gƒ}t|ƒdkr@|gS|g}d}xV|D]N}|j|||ƒ} |d7}|t|ƒkrš| jƒn|| 7}qVW|S(st get a ring structure that tends to share nodes with the tree return a list starting from r ii(R7R tfind_share_ringtreverse( RR>R?RDtnsettcsettrlsttcnttvtvlst((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyRdÁs    c C@s²|ddkst‚|j||dƒ}t|ƒt|ƒksIt‚i}t|ƒ}xPt|ƒD]B}||d|}|d|}||||f|||R?RhR@RWRDRBRC((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytget_ringÔs  c C@s_|j|ƒ\}}|j||ƒ}idd6}d}x4t|dƒD]"}||d}|d|||jƒD]0\}} || d|| df|||R?R@trmaptktit ring_map_t tree_map_t parent_map_Rjtx((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt get_link_mapãs& () cC@s¤i}i}i}g}d}xCt|ƒ|krc|jjƒ\}}t||ƒ} | jdkrŽ| jjƒ} tj| j ƒƒq!n| jdkr| j dkr»| j |ksÁt ‚| j |ksÖt ‚| || j +RR4s)Recieve %s signal from %s; assign rank %ds(@tracker All of %d nodes getting starteds@tracker All nodes finishes jobs2@tracker %s secs between node start and job finish(R#R RtacceptR(R.RRXRYtstripR+R*tdebugR,RutlistR:R6R tsortR<R-RLR/R$ttimeRURVtstr(RRWRwR=R5tpendingR>tfdR1RtmsgR?R@t todo_nodesR+((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt accept_slavesþsl$ $  $       c@sK‡‡fd†}td|ddƒˆ_ˆjjtƒˆjjƒdS(Nc@sˆjˆƒdS(N(R‡((RWR(s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytrunCsttargettargs((RRTt setDaemonR9Rx(RRWRˆ((RWRs5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyRxBscC@s*x#|jjƒr%|jjdƒqWdS(Nid(RTtisAliveR (R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyR IscC@s |jjƒS(N(RTRŒ(R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytaliveMs(RRR RR]t staticmethodR_RbRcRdRlRuR‡RxR R(((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyRM‰s       D  t PSTrackercB@s;eZdZdddd„Zd„Zd„Zd„ZRS(s Tracker module for PS iƒ#i'c @se|ˆ_|dkrdS|dkr+in|}|ˆ_tjt|ƒtjƒ}x\t||ƒD]K}y+|jd|fƒ|ˆ_|j ƒPWqetj k r¯qeqeXqeWt j j ƒ‰dˆdgethostbyname(socket.getfqdn()) failed... trying on hostname()s127.s10.255.255.255i(s10.255.255.255i( R#R!tgetfqdnR¡t gethostbynameRXtwarnt gethostnamet startswithtAF_INETt SOCK_DGRAMtconnectt getsockname(RSR¡R((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt get_host_ip…s       RžcC@s |dkrd}ni|d6|d6}t|ƒ}|dkr˜td|d|ƒ}|j|jƒƒ|j|ƒ|jƒrå||||ƒqånMtd|d|d|ƒ}|j|jƒƒ|jƒrå||||ƒn|dkrþ|jƒn |jƒdS(NitDMLC_NUM_WORKERtDMLC_NUM_SERVERRSRWR.Rœ( R#R«RMtupdateRbRxRRR (tnworkertnservert fun_submitRStpscmdRœtrabittpserver((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytsubmitšs$           cC@sÖi|jd6|jd6}tdt|jƒd|jƒ}|j|jƒƒ|j|jƒtj j dƒx:|j ƒD],\}}tj j d|t |ƒfƒq{Wtj j dƒtj j ƒ|jƒdS( sStandalone function to start rabit tracker. Parameters ---------- args: arguments to start the rabit tracker. R¬R­RSRWsDMLC_TRACKER_ENV_START s%s=%s sDMLC_TRACKER_ENV_END N(t num_workerst num_serversRMR«thost_ipR®RbRxtsyststdouttwriteRmR‚tflushR (RŠRœR³RoRj((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytstart_rabit_tracker³s  !$ c C@s;tjddƒ}|jddtdtddƒ|jdd d dtdd ƒ|jd d ddtdd dƒ|jdd ddtdddgddƒ|jƒ}d}|jdkrËt j }n.|jdkræt j }nt d|jƒ‚t j d|d|ƒ|jd kr+t|ƒn t dƒ‚dS(s8Main function if tracker is executed in standalone mode.t descriptionsRabit Tracker start.s --num-workerstrequiredttypethelps)Number of worker proccess to be launched.s --num-serverstdefaultis>Number of server process to be launched. Only used in PS jobs.s --host-ips'Host IP addressed, this is only needed s/if the host IP cannot be automatically guessed.s --log-leveltINFOtchoicestDEBUGsLogging level of the logger.s%%(asctime)s %(levelname)s %(message)ssUnknown logging level %stformattlevels7Do not yet support start ps tracker in standalone mode.N(targparsetArgumentParsert add_argumentR9tintR#R‚t parse_argst log_levelRXRÃRÅt RuntimeErrort basicConfigR·R½(tparserRŠtfmtRÇ((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pytmainÈs,     t__main__(R t __future__RR™R¹R!RR—RÈRRXt threadingRtobjectRR)R%R'R(RMRR#R«RµR½RÒR(((s5/tmp/pip-install-Qvdv_2/mxnet/dmlc_tracker/tracker.pyt s.          OÇ5