U C^W@sddlmZmZmZddlZddlmZmZddl m Z Gddde Z Gdd d e Z Gd d d e Zd d ZddZddZdddZddZdS))divisionprint_functionunicode_literalsN)tags_to_entities GoldParse)Errorsc@sDeZdZdZddZddZeddZedd Zed d Z d S) PRFScorez( A precision / recall / F score cCsd|_d|_d|_dS)Nr)tpfpfnselfr//tmp/pip-install-6_kvzl1k/spacy/spacy/scorer.py__init__szPRFScore.__init__cCsH|jt||7_|jt||7_|jt||7_dSN)r len intersectionr r rZcandgoldrrr score_setszPRFScore.score_setcCs|j|j|jdSN0.++)r r r rrr precisionszPRFScore.precisioncCs|j|j|jdSr)r r r rrrrecallszPRFScore.recallcCs$|j}|j}d||||dS)Nr)rr)rprrrrfscore!szPRFScore.fscoreN) __name__ __module__ __qualname____doc__rrpropertyrrrrrrrr s  r c@s,eZdZdZddZddZeddZdS) ROCAUCScorez An AUC ROC score. cCsg|_g|_d|_d|_dS)Ngr)goldscands saved_scoresaved_score_at_lenr rrrr-szROCAUCScore.__init__cCs|j||j|dSr)r'appendr&rrrrr3s zROCAUCScore.score_setcCs^t|j|jkr|jSzt|j|j|_Wn tk rJtd |_YnXt|j|_|jS)Ninf)rr&r)r(_roc_auc_scorer' ValueErrorfloatr rrrscore7s zROCAUCScore.scoreN)r r!r"r#rrr$r/rrrrr%(s r%c@seZdZdZd!ddZeddZedd Zed d Zed d Z eddZ eddZ eddZ eddZ eddZeddZeddZeddZd"dd ZdS)#ScorerzCompute evaluation scores.FNcCst|_t|_t|_t|_t|_t|_t|_t|_ ||_ d|_ t|_ d|_ d|_|r|D]\}}|dkrh|jdd|_ |j rt|_ |jddsd|_|jdgD]}t|j |<qqh|jdgD]}t|j |<qqhdS)zInitialize the Scorer. eval_punct (bool): Evaluate the dependency attachments to and from punctuation. RETURNS (Scorer): The newly created object. DOCS: https://spacy.io/api/scorer#init NFtextcatZpositive_labelZexclusive_classesTlabels)r tokensZsbd unlabelledlabelleddictlabelled_per_deptagsner ner_per_ents eval_punctr1textcat_per_cattextcat_positive_labeltextcat_multilabelcfggetr%)rr;ZpipelinenamemodellabelrrrrHs2  zScorer.__init__cCs |jjdS)zhRETURNS (float): Part-of-speech tag accuracy (fine grained tags, i.e. `Token.tag`). d)r8rr rrrtags_accmszScorer.tags_acccCs |jjdS)z'RETURNS (float): Tokenization accuracy.rD)r3rr rrr token_acctszScorer.token_acccCs |jjdS)z-RETURNS (float): Unlabelled dependency score.rD)r4rr rrruasysz Scorer.uascCs |jjdS)z+RETURNS (float): Labelled dependency score.rD)r5rr rrrlas~sz Scorer.lascCsdd|jDS)z5RETURNS (dict): Scores per dependency label. cSs0i|](\}}||jd|jd|jddqSrD)rrfrrr.0kvrrr sz'Scorer.las_per_type..)r7itemsr rrr las_per_typeszScorer.las_per_typecCs |jjdS)z3RETURNS (float): Named entity accuracy (precision).rD)r9rr rrrents_psz Scorer.ents_pcCs |jjdS)z0RETURNS (float): Named entity accuracy (recall).rD)r9rr rrrents_rsz Scorer.ents_rcCs |jjdS)z1RETURNS (float): Named entity accuracy (F-score).rD)r9rr rrrents_fsz Scorer.ents_fcCsdd|jDS)z1RETURNS (dict): Scores per entity label. cSs0i|](\}}||jd|jd|jddqSrIrKrLrrrrPsz(Scorer.ents_per_type..)r:rQr rrr ents_per_typeszScorer.ents_per_typecCsn|jsB|jr|jjdStdd|jDt|jddSttdd|jDt|jddS)zRETURNS (float): f-score on positive label for binary exclusive, macro-averaged f-score for 3+ exclusive, macro-averaged AUC ROC score for multilabel (-1 if undefined) rDcSsg|]\}}|jqSr)rrMrCr/rrr sz(Scorer.textcat_score..rcSsg|]\}}|jqSr)r/rWrrrrXs) r>r=r1rsumr<rQrmaxr rrr textcat_scores   zScorer.textcat_scorecCs.|jsdd|jDSdd|jDS)z2RETURNS (dict): Scores per textcat label. cSs0i|](\}}||jd|jd|jddqSrIrKrLrrrrPsz+Scorer.textcats_per_cat..cSs"i|]\}}|dt|jdiqS)Z roc_auc_scorerY)r[r/rLrrrrPs)r>r<rQr rrrtextcats_per_catszScorer.textcats_per_catc Cs2|j|j|j|j|j|j|j|j|j|j |j d S)zRETURNS (dict): All scores with keys `uas`, `las`, `ents_p`, `ents_r`, `ents_f`, `tags_acc`, `token_acc`, and `textcat_score`. rGrHrRrSrTrUrVrErFr\r]r^r rrrscoressz Scorer.scoresrpunctc&st|t|kr.t|tt|j|jf}t}i}t}ttdd|jD}|jD]\} } } } } }| | | f| dkr\| |kr\| | | | f| |j krt |j | <| |krt|| <||  | | | fq\t}i}t}|D]8}|j r"q |j|j}|dkrJ|jjd7_n |jjd7_| ||jf|j |kr |j r |j|jj}|dks|dkr|jjd7_|jjd7_nz| |||j f|j |j krt |j |j <|j |kr$t||j <||j  |||j fq ddd|jDkrtdd|Dd d|jD}d d |D}|D]:|jkrt |j<|fd d|Dqt}d d |D}|jD]}|j|j}|j|jd}|dks|dkrD|jjd7_|j|j jd7_n*| |j ||f||j  |j ||fq|j!D](\}}||kr||"||||q||j"|||j#"|||j"|||j D],} |j | "|$| t|$| tq|j"tdd|Dtdd|Dt|jdkr2t|j%t|jkr2t|jt|jkr2t&|j|jj$d}t&|j|jj$d}|j'r|j("t|j'gt|g@t|j'gt|g@|j%D]`} |j)r|j%| "|j| |j| n2|j%| "t| gt|g@t| gt|g@qn8t|j%dkrjt|j%}!t|j}"t*t+j,j-|!|"d|rdd|jD}#||D]"\}$}%} t.d|#|$| |#|%q||D]"\}$}%} t.d|#|$| |#|%qdS)aUpdate the evaluation scores from a single Doc / GoldParse pair. doc (Doc): The predicted annotations. gold (GoldParse): The correct annotations. verbose (bool): Print debugging information. punct_labels (tuple): Dependency labels for punctuation. Used to evaluate dependency attachments to punctuation if `eval_punct` is `True`. DOCS: https://spacy.io/api/scorer#score cSsg|] }|dqSrYr)rMZannotrrrrXsz Scorer.score..)NNr-cSsg|] }|dqSrbr)rMtokenrrrrXscSsg|] }|dqSrrrMxrrrrXscSsg|] }|jqSr)label_)rMrNrrrrXscSsi|] }|tqSrsetrM ent_labelrrrrPsz Scorer.score..csg|]}|dkr|qSrfrrgrmrrrXs cSsi|] }|tqSrrjrlrrrrP"scss|]}|ddVqdS)NrrrMitemrrr 9szScorer.score..r)key) model_labels eval_labelscSsg|] }|dqS)rrrorrrrXWsFM)/rrZfrom_annot_tuplestuplezipZ orig_annotZcatsrkraddlowerr7r Zorth_isspaceZ cand_to_goldir3r r Ztag_Zdep_stripheadr4r5Zentsr:updatestartendr9rirQrr8r@r<r[r=r1r>r-rZE162formatprint)&rdocrverboseZ punct_labelsZ gold_depsZgold_deps_per_depZ gold_tagsZ gold_entsZid_wordtagr~depr9Z cand_depsZcand_deps_per_depZ cand_tagsreZgold_iZ gold_headZ ent_labelsZ gold_per_entsZ cand_entsZ cand_per_entsentfirstlastrNrOZgoldcatZcandcatrCrsrtZ gold_wordsZw_idZh_idrrnrr/s      "               z Scorer.score)FN)Fr`)r r!r"r#rr$rErFrGrHrRrSrTrUrVr\r]r_r/rrrrr0Es6 %            r0cCs6tt|dkrttjt||\}}}t||S)aCompute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores. Note: this implementation is restricted to the binary classification task Parameters ---------- y_true : array, shape = [n_samples] or [n_samples, n_classes] True binary labels or binary label indicators. The multiclass case expects shape = [n_samples] and labels with values in ``range(n_classes)``. y_score : array, shape = [n_samples] or [n_samples, n_classes] Target scores, can either be probability estimates of the positive class, confidence values, or non-thresholded measure of decisions (as returned by "decision_function" on some classifiers). For binary y_true, y_score is supposed to be the score of the class with greater label. The multiclass case expects shape = [n_samples, n_classes] where the scores correspond to probability estimates. Returns ------- auc : float References ---------- .. [1] `Wikipedia entry for the Receiver operating characteristic `_ .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition Letters, 2006, 27(8):861-874. .. [3] `Analyzing a portion of the ROC curve. McClish, 1989 `_ r)rnpuniquer-rZE165 _roc_curve_auc)y_truey_scorefprtpr_rrrr,s$ r,cCst||\}}}tjd|f}tjd|f}tj|dd|f}|ddkr`ttj|j}n ||d}|ddkrttj|j}n ||d}|||fS)aCompute Receiver operating characteristic (ROC) Note: this implementation is restricted to the binary classification task. Parameters ---------- y_true : array, shape = [n_samples] True binary labels. If labels are not either {-1, 1} or {0, 1}, then pos_label should be explicitly given. y_score : array, shape = [n_samples] Target scores, can either be probability estimates of the positive class, confidence values, or non-thresholded measure of decisions (as returned by "decision_function" on some classifiers). Returns ------- fpr : array, shape = [>2] Increasing false positive rates such that element i is the false positive rate of predictions with score >= thresholds[i]. tpr : array, shape = [>2] Increasing true positive rates such that element i is the true positive rate of predictions with score >= thresholds[i]. thresholds : array, shape = [n_thresholds] Decreasing thresholds on the decision function used to compute fpr and tpr. `thresholds[0]` represents no instances being predicted and is arbitrarily set to `max(y_score) + 1`. Notes ----- Since the thresholds are sorted from low to high values, they are reversed upon returning them to ensure they correspond to both ``fpr`` and ``tpr``, which are sorted in reversed order during their calculation. References ---------- .. [1] `Wikipedia entry for the Receiver operating characteristic `_ .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition Letters, 2006, 27(8):861-874. rrrY)_binary_clf_curverr_repeatnanshape)rrfpstpsZ thresholdsrrrrrrs.    rc Csd}t|}t|}||k}tj|ddddd}||}||}d}tt|d}tj||jdf}t|||}d||}||||fS)aCalculate true and false positives per binary classification threshold. Parameters ---------- y_true : array, shape = [n_samples] True targets of binary classification y_score : array, shape = [n_samples] Estimated probabilities or decision function Returns ------- fps : array, shape = [n_thresholds] A count of false positives, at index i being the number of negative samples assigned a score >= thresholds[i]. The total number of negative samples is equal to fps[-1] (thus true negatives are given by fps[-1] - fps). tps : array, shape = [n_thresholds <= len(np.unique(y_score))] An increasing count of true positives, at index i being the number of positive samples assigned a score >= thresholds[i]. The total number of positive samples is equal to tps[-1] (thus false negatives are given by tps[-1] - tps). thresholds : array, shape = [n_thresholds] Decreasing score values. g?Z mergesort)kindNrYrr)rravelZargsortwherediffrsize_stable_cumsum) rrZ pos_labelZdesc_score_indicesZweightZdistinct_value_indicesZthreshold_idxsrrrrrrs   rh㈵>:0yE>c CsVtj||tjd}tj||tjd}ttj|jd|d|||ddsRttj |S)aUse high precision for cumsum and check that final value matches sum Parameters ---------- arr : array-like To be cumulatively summed as flat axis : int, optional Axis along which the cumulative sum is computed. The default (None) is to compute the cumsum over the flattened array. rtol : float Relative tolerance, see ``np.allclose`` atol : float Absolute tolerance, see ``np.allclose`` )axisdtyperY)rT)rtolatolZ equal_nan) rZcumsumZfloat64rZalliscloseZtaker-rZE163)Zarrrrroutexpectedrrrr's  rcCst|}t|}d}t|}t|dkrTt|dkrDd}nttj||t ||}t |tj r||j |}|S)aCompute Area Under the Curve (AUC) using the trapezoidal rule This is a general function, given points on a curve. For computing the area under the ROC-curve, see :func:`roc_auc_score`. Parameters ---------- x : array, shape = [n] x coordinates. These must be either monotonic increasing or monotonic decreasing. y : array, shape = [n] y coordinates. Returns ------- auc : float rrrY)rrranyrr-rZE164rZtrapz isinstanceZmemmaprtype)rhy directionZdxarearrrrAs     r)Nrr) __future__rrrZnumpyrrrrerrorsrobjectr r%r0r,rrrrrrrrs A*C6