U C^@sFddlmZddlZddlmZddlmZmZd dd Zd d Z dS) )unicode_literalsN) docs_to_json)get_lang_class minibatch Fc s|dkrtdg}dd|dD}t|}|d}tt||dD]\} } g} | D]l} | d} d| kr~| d}n| d }d d|D}|| |fd d|D}t|_ | q`| t | | d qP|S) Nz.No --lang specified, but tokenization requiredcSsg|]}t|qS)srslyZ json_loads).0linerrB/tmp/pip-install-6_kvzl1k/spacy/spacy/cli/converters/jsonl2json.py sz"ner_jsonl2json.. sentencizer)sizetextentitiesspanscSs"g|]}|d|d|dfqS)startendlabelr)r errr r scs"g|]\}}}j|||dqS))r)Z char_span)r srLdocrr r s)id) ValueErrorstripsplitrZ create_pipe enumeraterZmake_doc_cleanup_spansentsappendr)Z input_datalangZn_sentsZuse_morphology_Z json_docsZinput_examplesZnlpribatchZdocsrecordZraw_textr"rrrr ner_jsonl2json s*      r)cCsg}t}|D]}|dk rt|r:|djr:|dd}qt|rZ|djrZ|dd}q:t|sdqt|j|jD]}||krrqqr|||t|j|jq|S)Nr)setlenZis_spacerangerrr#update)routputseenspanr&rrr r!#s  r!)NrF) __future__rr Zgoldrutilrrr)r!rrrr s