U C^>@s2ddlmZddlZddlZddlmZddlmZddlm Z d>ddZ d d Z e d?d d Z e d@ddZ dAddZddZddZddZddZddZddZddZd d!ZdBd"d#Zd$d%Zd&d'ZdCd)d*Zd+d,ZdDd-d.Zd/d0ZdEd1d2ZdFd4d5ZdGd6d7Z d8d9Z!dHd:d;Z"d.wrapperr)r rr rrrr rlayerize srcsfdd}|S)zDWrap a function over a sequence of layers and an input into a layer.csfdd}t|f||S)Ncs|f||Sr r)Xr r)layers user_funcrrr sz4metalayerize..returned..begin_updater)rr rr rrrreturnedszmetalayerize..returnedr)rrrrr metalayerizes rcsJtjjdd|Ddddfdd }j|d}|f|fS) NcSsg|] }t|qSrlen.0seqrrr &sz'flatten_add_lengths..idtypecsj|dS)Npad) unflatten)d_Xsgdlengthsopsr%rr finish_update(sz*flatten_add_lengths..finish_updater$)N)rr+asarrayflatten)seqsr%dropr,rrr)rflatten_add_lengths#s r1cs0tj|\}}||}dfdd }||fS)Ncsj|dd}|S)Nrr$)r.)ZdXsr(dXr+rrbackprop_unflatten4sz%unflatten..backprop_unflatten)N)rr+r&)Z X_lengthsr0rr*Xsr4rr3rr&.s  r&cs4ddidfdd }t|}dkr0|j|S)Nrrcsx|ddf}t|tjs$|}t}t|D]4\}}t|}|kr\||<|d7}|||<q4|dfS)Nr) isinstancenumpyndarraygetr enumerateintr-)idsr0Zn_vectorr!Zid_columnZid_mapr+rr remap_ids_fwd=s z remap_ids..remap_ids_fwd)r)rr+)r+r>r?modelrr=r remap_ids:s  rAcsdfdd }t|S)Nrcs|jtddjg}|jd|jd|d|jd}|jjj}j||d\}||}dfdd }||fS) Nrrr0cs.|djjj}||dS)NrBr()reshapeastyper+xpfloat32)dYr() Y2d_backprop initial_shapelayernBnTrrwith_reshape_backward[szIwith_reshape..with_reshape_forward..with_reshape_backward)N) shapelistZnOrFrGr+rHrIr )rr0Z final_shapeZX2dZY2dYrPrM)rKrLrNrOrwith_reshape_forwardQs   z*with_reshape..with_reshape_forward)rwrap)rMrUrrTr with_reshapePsrXcsBdfdd }t|}|jfdd}|j||S)Nrcs<j||d\}}|d|f|dd|fS)NrDrr )itemsr0rfinishidxrMrrr esz"with_getitem..begin_updatecs jD]}|||qdSr  on_data_hooks)selfrYyhookr[rron_datals zwith_getitem..on_data)r)r_layersappendr^)r\rMr r@rbrr[r with_getitemds   recGsddd}|S)z6Transform a sequences of layers into a null operation.rcSs |ddfS)Nc_s|Sr r)Dakrrrxz,noop..begin_update..rrr0rrrr wsznoop..begin_update)rr)rr rrrnoopts rlcGs4t|dkrtgSt|dkr(|dSt|SdS)zCompose two models `f` and `g` such that they become layers of a single feed-forward model that computes `g(f(x))`. Raises exception if their dimensions don't match. rrN)rrrrrrchain}s   rmcCsN|dkrttS|g}t|dD] }|t||dq$t|S)zqConstruct `n` copies of a layer, with distinct weights. i.e. `clone(f, 3)(x)` computes `f(f'(f''(x)))`. rrrB)rrlrangerdcopydeepcopyZset_idr)orignrr!rrrclones rscsNs tSdjfdd}t|}t|_ddd}|j||S)zCompose two or more models `f`, `g`, etc, such that their outputs are concatenated, i.e. `concatenate(f, g)(x)` computes `hstack(f(x), g(x))` rcsVt\}fdd|D}j|}dd|Dfdd}||fS)Ncsg|]}|fqSrr)rZfwd)rrgrhrrr sz5concatenate..begin_update..cSsg|] }|jqSrrQ)rvalrrrr sc sg}d}tD]z\}}||d}|dk r|j|dd||ff||}|dk rtdr|sx||n|d|7<|}q|r|dSdSdS)NrrrQrB)ziprHascontiguousarrayhasattrrdr-) Zgradientr rZ layer_gradsstartZbwdrQendd)rbackwardr+shapesrrr,s(  z8concatenate..begin_update..finish_update)split_backwardrHZhstack)rrgrhforwardvaluesoutputr,rr+)rrgr|rhr}rr s   z!concatenate..begin_updateNcSs(|jD]}|jD]}||||qqdSr rcr^r_rr`rMrarrrrbs  zconcatenate..on_data)N)rlr+rrRrcr^rd)rr rMrbrrr concatenates    rcsFs tSdfdd }t|}t|_fdd}|j||S)NrcsLtfddD\}|d}|D] }||7}q(dfdd }||fS)Ncsg|]}|jdqS)rDr )rZlyrrkrrr sz(add..forward..rcsLfddD}dd|D}|rD|d}|D] }||7}q2|SdSdS)Ncs g|]}|dk r|dqSNrEr)rZbpd_outr(rrr sz:add..forward..backward..cSsg|]}|dk r|qSr r)rgrrrr srr)rr(Zgradstotalr) callbacksrrr|s z&add..forward..backward)N)rv)rr0Zoutsoutor|r)rrr0rrs   zadd..forwardcs&D]}|jD]}||||qqdSr r]rrrrrbs zadd..on_data)r)rlrrRrcr^rd)rrr@rbrrradds   rcsgfdd|D}|fS)aSeparate a sequence of layers' `begin_update` methods into two lists of functions: one that computes the forward values, and the other that completes the backward pass. The backward sequence is only populated after the forward functions have been applied. csg|]}t|jjqSr) sink_returnr rd)ropr|rrr sz"split_backward..r)rrrrrr~sr~csfdd}|S)zTransform a function `func` that returns tuples into a function that returns single values. Call a function `sink` on the unused values. cs4||}dkr|\}}n |\}}||Sr r)r rrZto_keepZto_sinkfuncsinksplitterrrrWs    zsink_return..wrapr)rrrrWrrrrs rcstdfdd }|S)Nrcstt|}|dfSr )rRrv)Zbatched_inputsr0inputsr!rrr s zArg..begin_update)rr)r!r rrrArg srcsdfdd }t|S)NrcsHj|\}}}j||d\\}}||}dfdd }||fS)NrDcs*j|\}}}|df|d}||Sr)r+square_sequences) d_seqs_outr(Z d_padded_outZ sizes_at_tunpadZ d_padded_in)backprop_modelr@rrbackprop_paddingszGwith_square_sequences..padded_forward..backprop_padding)N)r+rr )seqs_inr0Z padded_in_rZ padded_outZseqs_outrr@)rrpadded_forwards z-with_square_sequences..padded_forward)rrV)r@rrrrwith_square_sequencess rcsLdfdd }fdd}t||d}|j|jtd|_|S) Nrcs|jdd|Djjj|d|d\}dkrRjj|ddfSdfdd }jj|d|fS)NcSsg|] }t|qSrrrrrrr &sz6with_flatten..begin_update..r$rDcs:jj|d|d}|dkr$dSjj|dSdS)Nr$rEr+r.r&)rr(r')bp_layerrMr*r%rrr,+sz9with_flatten..begin_update..finish_update)N)r+r-r r.r&)rr0rr,rMr%)rr*rr %s z"with_flatten..begin_updatecs<jdd|D}jj|d}jj||dS)NcSsg|] }t|qSrrrrrrr 5sz1with_flatten..predict..r$)r+r-r.r&)rr*rrrrr4szwith_flatten..predict)rr.)r)rrcrdr^_with_flatten_on_dataname)rMr%ndimr rr@rrr with_flatten$s   rcCs<|j|}|jD]$}|jD]}||||q||}qdSr )r+r.rcr^r@rr`rMrarrrr@s    rcsdfdd }t|S)NrcsXg}tj}|D]@}dk r,fdd|D}dd|D}||j|ddq|dfS)zGet word forms.Ncsg|]}|s|qSrrrtokenignorerrr Psz1get_word_ids..forward..cSsg|]}|jp|jqSr)Zlex_idZorthrrrrr RsZuint64r")rr+rdr-)docsr0r/r+docrrrrrJszget_word_ids..forward)rr)r+r%Z token_droprrrrr get_word_idsHs rcGs,t|}|j|dd}|j||S)NcSs(|jD]}|jD]}||||qqdSr r)r_rr`childrarrrrb^s  zwrap..on_data)rrcextendr^rd)rZ child_layersr@rbrrrrWZs   rWcsdfdd }t|}|S)aGroup inputs to a layer, so that the layer only has to compute for the unique values. The data is transformed back before output, and the same transformation is applied for the gradient. Effectively, this is a cache local to each minibatch. The uniqued wrapper is useful for word inputs, because common words are seen often, but we may want to compute complicated features for the words, using e.g. character LSTM. rcs|ddf}jj|}t|tjs2|}tj|dddd\}}jj||}j||d\ |j dfj dd}dfdd }||fS) NT)Z return_indexZreturn_inverseZ return_countsrDrrcsZjjjdd}j|jjdd|||d}|dk rR|}|SdSdS)Nfr"r!rE)r+allocaterQZ scatter_addr-)rJr(ZdY_uniqZ d_uniquesr2)Y_uniq bp_Y_uniqcountsinvrMrr uniqued_bwd~s  z1uniqued..uniqued_fwd..uniqued_bwd)N) r+rHrwr6r7r8r9uniquer rFrQ)rr0keysZ uniq_keysindZX_uniqrSrr>rM)rrrrr uniqued_fwdrs $ zuniqued..uniqued_fwd)rrV)rMr>rr@rrruniquedgs  r?cs.dfdd }t|}dd}|g|_|S)zMap a layer across list itemsrc sg}g|D]}dd|D}t|s*trDfdd|D}nt|}|rj||t|q tj|||ddq t|stj|dd\}j |}d fdd }||fS) NcSsg|]}t|r|qSrrrsentrrrr sz0foreach..foreach_fwd..cs"g|]}tjkr|qSrr7randomrsr0 drop_factorrrr srrrrDcs6j|}||d}|dkr$|Sj|SdSrrZd_outputr(Zd_flatZd_sentsbp_flatrMr*rr foreach_bwds   z1foreach..foreach_fwd..foreach_bwd)N) rAssertionErrorrRrrdr7rshuffler r+r&) rr0sentsr doc_sentssubsetflatrrrrMrr0r*r foreach_fwds*      zforeach..foreach_fwdcSs0|jD]$}|jD]}|||d|dqqdSNrrrrrr_run_foreach_child_hookss  z)foreach.._run_foreach_child_hooks)r)rWr^)rMrrr@rrrrforeachs ! rcsdfdd }t|}|S)zCMap a layer across sentences (assumes spaCy-esque .sents interface)rc sg}g|D]j}dd|jD}fdd|D}|rR||t|q tj|||ddq j|dd\}j |}d fdd }||fS) NcSsg|]}t|r|qSrrrrrrr sz:foreach_sentence..sentence_fwd..cs"g|]}tjkr|qSrrrrrrr srrrrDcs6j|}||d}|dkr$|Sj|SdSrrrrrr sentence_bwds   z.sentence_fwd..sentence_bwd)N) rrrdrr7rrr r+r&) rr0rrrrrrrrrr sentence_fwds"    z&foreach_sentence..sentence_fwd)rrV)rMrrr@rrrforeach_sentences rcsdfdd }t|S)Nrcs~jjdd|Ddd}|tj|d\}t||}j|d|df|d\}d fdd }||fS) NcSsg|]}|jdqS)rrt)rxrrrr szIwith_pad_and_mask..create_model_input_forward..r!r"pad_torIrDcs(tj|d\}}||d}|S)NrrE) pad_sequencesr+)ZdYsr(rJrr2)bp_Yr@nLunpad_Xrrcreate_model_input_backwards zZwith_pad_and_mask..create_model_input_forward..create_model_input_backward)N)r+r-maxr _get_maskr rG)r5r0nXrX_maskrSrrMr@)rrrrcreate_model_input_forwards z5with_pad_and_mask..create_model_input_forward)rr)rMrrrrwith_pad_and_masks rcs|jdd|Dddt|}|dkr0}|j|t|f|djdd|djd}t|D]$\}}||||d|jdf<qffdd}||fS) NcSsg|] }t|qSrrrrrrr sz!pad_sequences..r!r"rrcs>dgt}t|jdD]}||d|f||<q|Sr)rrnrQ)ZpaddedZunpaddedr!r*rrrszpad_sequences..unpad)r-rrrr;rQr#r:)r+rrrNZarrZarr_irrrrrrs. rcCsR|jd}|jd}tj|||f}t|D]\}}d||ddd|f<q.|S)Nrrr)rQrr+rr:)rrrNrrr!lengthrrrrs   r)NN)rr)r)Nr)N)rr)rrN)r)r)r)N)$ __future__rror7Zneural._classes.modelrZneural._classes.function_layerrZneural._classes.feed_forwardrrrr1r&rArXrerlrmrsrrr~rrrrrrrWrrrrrrrrrrsB         3#     ' 0 %