B e]>@s2ddlmZddlZddlZddlmZddlmZddlm Z d>ddZ d d Z e d?d d Z e d@ddZ dAddZddZddZddZddZddZddZddZd d!ZdBd"d#Zd$d%Zd&d'ZdCd)d*Zd+d,ZdDd-d.Zd/d0ZdEd1d2ZdFd4d5ZdGd6d7Z d8d9Z!dHd:d;Z"d.wrapper)r)rrr r r r )r r r layerize srcsfdd}|S)zDWrap a function over a sequence of layers and an input into a layer.csfdd}t|f||S)Ncs|f||S)Nr )Xr r )layers user_funcr r rsz4metalayerize..returned..begin_update)r)rr r r)r)rr returnedszmetalayerize..returnedr )rrr )rr metalayerizes rcsJtjjdd|Ddddfdd }j|d}|f|fS) NcSsg|] }t|qSr )len).0seqr r r &sz'flatten_add_lengths..i)dtypecsj|dS)N)pad) unflatten)d_Xsgd)lengthsopsrr r finish_update(sz*flatten_add_lengths..finish_update)r)N)rr asarrayflatten)seqsrdropr!rr )rr rr flatten_add_lengths#s r&cs0tj|\}}||}dfdd }||fS)Ncsj|dd}|S)Nr)r)r#)ZdXsrdX)r r r backprop_unflatten4sz%unflatten..backprop_unflatten)N)rr r)Z X_lengthsr%rrXsr(r )r r r.s  rcs4ddidfdd }t|}dkr0|j|S)Nrcs||ddf}t|tjs$|}t}x@t|D]4\}}t|}|kr^||<|d7}|||<q6W|dfS)Nr) isinstancenumpyndarraygetr enumerateintr")idsr%Zn_vectorrid_)columnid_mapr r r remap_ids_fwd=s z remap_ids..remap_ids_fwd)r*)rr )r r3r5modelr )r3r4r r remap_ids:s  r7csdfdd }t|S)Ncs|jtddjg}|jd|jd|d|jd}|jjj}j||d\}||}dfdd }||fS) Nrr)r%cs.|djjj}||dS)Nr9)r)reshapeastyper xpfloat32)dYr) Y2d_backprop initial_shapelayernBnTr r with_reshape_backward[szIwith_reshape..with_reshape_forward..with_reshape_backward)N) shapelistZnOr;r<r r=r>r)rr%Z final_shapeZX2dZY2dYrE)rB)r@rArCrDr with_reshape_forwardQs   z*with_reshape..with_reshape_forward)r8)wrap)rBrIr )rBr with_reshapePsrKcsBdfdd }t|}|jfdd}|j||S)Ncs<j||d\}}|d|f|dd|fS)N)r%r)r)itemsr%rfinish)idxrBr r resz"with_getitem..begin_updatecs$xjD]}|||qWdS)N) on_data_hooks)selfrMyhook)rOrBr r on_datals zwith_getitem..on_data)rL)r_layersappendrP)rOrBrr6rTr )rOrBr with_getitemds   rWcGsddd}|S)z6Transform a sequences of layers into a null operation.cSs |ddfS)Nc_s|S)Nr )Dakr r r xz,noop..begin_update..r )rr%r r r rwsznoop..begin_update)rXr )rrr r r noopts r^cGs4t|dkrtgSt|dkr(|dSt|SdS)zCompose two models `f` and `g` such that they become layers of a single feed-forward model that computes `g(f(x))`. Raises exception if their dimensions don't match. rrN)rr)rr r r chain}s   r_cCsR|dkrttS|g}x0t|dD] }|t||dq&Wt|S)zqConstruct `n` copies of a layer, with distinct weights. i.e. `clone(f, 3)(x)` computes `f(f'(f''(x)))`. rrr9)rr^rangerVcopydeepcopyset_idr)orignrrr r r clones rfcsNs tSdjfdd}t|}t|_ddd}|j||S)zCompose two or more models `f`, `g`, etc, such that their outputs are concatenated, i.e. `concatenate(f, g)(x)` computes `hstack(f(x), g(x))` rcsVt\}fdd|D}j|}dd|Dfdd}||fS)Ncsg|]}|fqSr r )rZfwd)rrZr[r r rsz5concatenate..begin_update..cSsg|] }|jqSr )rF)rvalr r r rsc sg}d}xtD]z\}}||d}|dk r|j|dd||ff||}|dk rtdr|sz||n|d|7<|}qW|r|dSdSdS)NrrrFr9)zipr=ascontiguousarrayhasattrrVr") Zgradientr r Z layer_gradsstartZbwdrFendd)rbackwardr shapesr r r!s"  z8concatenate..begin_update..finish_update)split_backwardr=Zhstack)rrZr[forwardvaluesoutputr!)rr )rrZrnr[ror rs   z!concatenate..begin_updateNcSs0x*|jD] }x|jD]}||||qWqWdS)N)rUrP)rQrrRrBrSr r r rTs  zconcatenate..on_data)N)r^r rrGrUrPrV)rrrBrTr )rr r concatenates    rtcsFs tSdfdd }t|}t|_fdd}|j||S)NcsPtfddD\}|d}x|D] }||7}q*Wdfdd }||fS)Ncsg|]}|jdqS))r%)r)rZlyr)rr%r r rsz(add..forward..rcsPfddD}dd|D}|rH|d}x|D] }||7}q4W|SdSdS)Ncs g|]}|dk r|dqS)N)rr )rZbp)d_outrr r rsz:add..forward..backward..cSsg|]}|dk r|qS)Nr )rgr r r rsrr )rvrZgradstotalrw) callbacks)rvrr rns  z&add..forward..backward)N)rh)rr%Zoutsoutorn)r)rryr%r rqs    zadd..forwardcs.x(D] }x|jD]}||||qWqWdS)N)rP)rQrrRrBrS)rr r rTs  zadd..on_data)ru)r^rrGrUrPrV)rrqr6rTr )rr adds   r|csgfdd|D}|fS)aSeparate a sequence of layers' `begin_update` methods into two lists of functions: one that computes the forward values, and the other that completes the backward pass. The backward sequence is only populated after the forward functions have been applied. csg|]}t|jjqSr ) sink_returnrrV)rop)rnr r rsz"split_backward..r )rrqr )rnr rpsrpcsfdd}|S)zTransform a function `func` that returns tuples into a function that returns single values. Call a function `sink` on the unused values. cs4||}dkr|\}}n |\}}||S)Nr )r r rsZto_keepZto_sink)funcsinksplitterr r rJs    zsink_return..wrapr )rrrrJr )rrrr r}s r}cstdfdd }|S)Ncstt|}|dfS)N)rGrh)Zbatched_inputsr%inputs)rr r r s zArg..begin_update)r)r)rrr )rr Arg srcsdfdd }t|S)NcsHj|\}}}j||d\\}}||}dfdd }||fS)N)r%cs*j|\}}}|df|d}||S)N)r)r square_sequences) d_seqs_outrZ d_padded_outZ sizes_at_tunpadZ d_padded_in)backprop_modelr6r r backprop_paddingszGwith_square_sequences..padded_forward..backprop_padding)N)r rr)seqs_inr%Z padded_in_rZ padded_outZseqs_outr)r6)rr padded_forwards z-with_square_sequences..padded_forward)r)rJ)r6rr )r6r with_square_sequencess rcsLdfdd }fdd}t||d}|j|jtd|_|S) Ncs|jdd|Djjj|d|d\}dkrRjj|ddfSdfdd }jj|d|fS)NcSsg|] }t|qSr )r)rrr r r r&sz6with_flatten..begin_update..)r)r%cs:jj|d|d}|dkr$dSjj|dSdS)N)r)r)r r#r)rrr)bp_layerrBrrr r r!+sz9with_flatten..begin_update..finish_update)N)r r"rr#r)rr%rr!)rBr)rrr r%s z"with_flatten..begin_updatecs<jdd|D}jj|d}jj||dS)NcSsg|] }t|qSr )r)rrr r r r5sz1with_flatten..predict..)r)r r"r#r)rrr)rBrr r r4szwith_flatten..predict)rr#)r)rrUrVrP_with_flatten_on_dataname)rBrndimrrr6r )rBrr with_flatten$s   rcCsD|j|}x2|jD](}x|jD]}||||q W||}qWdS)N)r r#rUrP)r6rrRrBrSr r r r@s    rcsdfdd }t|S)Ncs\g}tj}xH|D]@}dk r.fdd|D}dd|D}||j|ddqW|dfS)zGet word forms.Ncsg|]}|s|qSr r )rtoken)ignorer r rPsz1get_word_ids..forward..cSsg|]}|jp|jqSr )Zlex_idZorth)rrr r r rRsZuint64)r)rr rVr")docsr%r$r docr)rr r rqJs zget_word_ids..forward)r)r)r rZ token_droprrqr )rr get_word_idsHs rcGs,t|}|j|dd}|j||S)NcSs0x*|jD] }x|jD]}||||qWqWdS)N)rUrP)rQrrRchildrSr r r rT^s  zwrap..on_data)rrUextendrPrV)rZ child_layersr6rTr r r rJZs   rJcsdfdd }t|}|S)aGroup inputs to a layer, so that the layer only has to compute for the unique values. The data is transformed back before output, and the same transformation is applied for the gradient. Effectively, this is a cache local to each minibatch. The uniqued wrapper is useful for word inputs, because common words are seen often, but we may want to compute complicated features for the words, using e.g. character LSTM. cs|ddf}jj|}t|tjs2|}tj|dddd\}}jj||}j||d\ |j dfj dd}dfdd }||fS) NT)Z return_indexZreturn_inverseZ return_counts)r%rrcsZjjjdd}j|jjdd|||d}|dk rR|}|SdSdS)Nf)rr)r)r allocaterFZ scatter_addr")r?rZdY_uniqZ d_uniquesr')Y_uniq bp_Y_uniqcountsinvrBr r uniqued_bwd~s  z1uniqued..uniqued_fwd..uniqued_bwd)N) r r=rir+r,r-r.uniquerr;rF)rr%keysZ uniq_keysindZX_uniqrHr)r3rB)rrrrr uniqued_fwdrs $ zuniqued..uniqued_fwd)r)rJ)rBr3rr6r )r3rBr uniquedgs  r?cs.dfdd }t|}dd}|g|_|S)zMap a layer across list itemsc sg}gx|D]}dd|D}t|s,trFfdd|D}nt|}|rl||t|qtj|||ddqWt|stj|dd\}j |}d fdd }||fS) NcSsg|]}t|r|qSr )r)rsentr r r rsz0foreach..foreach_fwd..cs"g|]}tjkr|qSr )r,random)rs)r% drop_factorr r rsrrg)r%cs6j|}||d}|dkr$|Sj|SdS)N)r)r r#r)d_outputrd_flatd_sents)bp_flatrBrr r foreach_bwds   z1foreach..foreach_fwd..foreach_bwd)N) rAssertionErrorrGrrVr,rshufflerr r) rr%sentsr doc_sentssubsetflatrsr)rrB)rr%rr foreach_fwds&     zforeach..foreach_fwdcSs8x2|jD](}x"|jD]}|||d|dqWqWdS)Nr)rUrP)r6rrRrBrSr r r _run_foreach_child_hookss  z)foreach.._run_foreach_child_hooks)r)rJrP)rBrrr6rr )rrBr foreachs ! rcsdfdd }t|}|S)zCMap a layer across sentences (assumes spaCy-esque .sents interface)c sg}gxr|D]j}dd|jD}fdd|D}|rT||t|qtj|||ddqWj|dd\}j |}d fdd }||fS) NcSsg|]}t|r|qSr )r)rrr r r rsz:foreach_sentence..sentence_fwd..cs"g|]}tjkr|qSr )r,r)rr)r%rr r rsrrg)r%cs6j|}||d}|dkr$|Sj|SdS)N)r)r r#r)rrrr)rrBrr r sentence_bwds   z.sentence_fwd..sentence_bwd)N) rrrVrr,rrrr r) rr%rrrrrrsr)rrB)rr%rr sentence_fwds   z&foreach_sentence..sentence_fwd)r)rJ)rBrrr6r )rrBr foreach_sentences rcsdfdd }t|S)Ncs~jjdd|Ddd}|tj|d\}t||}j|d|df|d\}d fdd }||fS) NcSsg|]}|jdqS)r)rF)rxr r r rszIwith_pad_and_mask..create_model_input_forward..r)r)pad_tor>)r%cs(tj|d\}}||d}|S)N)r)r) pad_sequencesr )ZdYsrr?rr')bp_Yr6nLunpad_Xr r create_model_input_backwards zZwith_pad_and_mask..create_model_input_forward..create_model_input_backward)N)r r"maxr _get_maskrr<)r)r%nXrX_maskrHr)rBr6)rrrr create_model_input_forwards z5with_pad_and_mask..create_model_input_forward)r)r)rBrr )rBr6r with_pad_and_masks rcs|jdd|Dddt|}|dkr0}|j|t|f|djdd|djd}x0t|D]$\}}||||d|jdf<qhWfdd}||fS) NcSsg|] }t|qSr )r)rrr r r rsz!pad_sequences..r)rrrcsBdgt}x.t|jdD]}||d|f||<qW|S)Nr)rr`rF)ZpaddedZunpaddedr)rr r rszpad_sequences..unpad)r"rrrr0rFrr/)r rrrCZarrZarr_irrr )rr rs.  rcCsV|jd}|jd}tj|||f}x*t|D]\}}d||ddd|f<q0W|S)Nrrg?)rFrr rr/)rrrCrrrlengthr r r rs   r)NN)rr)r)Nr)N)rr)rrN)r)r)r)N)$ __future__rrar,Zneural._classes.modelrZneural._classes.function_layerrZneural._classes.feed_forwardrrrr&rr7rKrWr^r_rfrtr|rpr}rrrrrrJrrrrrrr r r r sB         3#     ' 0 %