B .(b@sdddlmZddlZddlmZddlmZmZmZddl m Z ddZ dd Z d d Z d d ZdS))unicode_literalsN)English) conllu2jsoniob2jsonconll_ner2json) make_docscCsRddddg}d|}t|dd}t|dks2t|dd dksFtt|dd dks^tt|dd dd dks~t|dd dd d}t|d d kst|d }dd|Dddddgkstdd|Dddddgkstdd|Dddddgkstdd|Dddddgks.td d|Dd!d"d#d!gksNtdS)$NzG1 Dommer dommer NOUN _ Definite=Ind|Gender=Masc|Number=Sing 2 appos _ Oz/2 Finn Finn PROPN _ Gender=Masc 4 nsubj _ B-PERz.3 Eilertsen Eilertsen PROPN _ _ 2 name _ I-PERuC4 avstår avstå VERB _ Mood=Ind|Tense=Pres|VerbForm=Fin 0 root _ O )n_sentsrid paragraphs sentencestokenscSsg|] }|dqS)orth).0trrC/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/test_cli.py sz3test_cli_converters_conllu2json..ZDommerZFinnZ EilertsenuavstårcSsg|] }|dqS)tagr)rrrrrrsZNOUNZPROPNZVERBcSsg|] }|dqS)headr)rrrrrrscSsg|] }|dqS)depr)rrrrrrsZapposZnsubjnameROOTcSsg|] }|dqS)nerr)rrrrrr sOzB-PERzL-PER)joinrlenAssertionError)lines input_data convertedsentrrrrtest_cli_converters_conllu2json s$     r&c Csddddg}d|}t|dd}t|dks2t|d d d ksFtt|d d dks^tt|d d d d d ks~txtd d D]}|d d d d |}t|ddkst|d}dd|Dddddddddgkstdd|DddddddddgkstqWdS) NzAI|O like|O London|I-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|OzAI|O like|O London|B-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|Oz^I|PRP|O like|VBP|O London|NNP|I-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|Oz^I|PRP|O like|VBP|O London|NNP|B-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|Or )r r rr r r rrcSsg|] }|dqS)rr)rrrrrr5sz0test_cli_converters_iob2json..IlikeLondonandNewYorkCity.cSsg|] }|dqS)rr)rrrrrr6srzU-GPEzB-GPEzI-GPEzL-GPE)rrr r!range)r"r#r$ir%rrrrtest_cli_converters_iob2json#s    &r3c.Csvddddddddd d dd d d ddddddddddddddddddddd d!d"dd#d$d%d&d'd(d)d*g.}d+|}t|d,d-}t|t|d.kst|d/d0d/kstt|d/d1d.kstt|d/d1d/d2d3kstxtd/d3D]}|d/d1d/d2|}t|d4d5kst|d4}d6d7|Dd8d9d:d;dd?gksHtd@d7|DdAdAdBdAdCdDdEdAgkstqWdS)FNz-DOCSTART- -X- O OzI Ozlike Oz London B-GPEzand Oz New B-GPEz York I-GPEz City I-GPEz. OzI Ozlike Oz London B-GPEzand Oz New B-GPEz York I-GPEz City I-GPEz. OzI PRP Oz like VBP OzLondon NNP B-GPEzand CC Oz New NNP B-GPEzYork NNP I-GPEzCity NNP I-GPEz. . Oz I PRP _ Oz like VBP _ OzLondon NNP _ B-GPEz and CC _ OzNew NNP _ B-GPEzYork NNP _ I-GPEzCity NNP _ I-GPEz. . _ Oz I PRP _ Oz like VBP _ OzLondon NNP _ B-GPEz and CC _ OzNew NNP _ B-GPEzYork NNP _ I-GPEzCity NNP _ I-GPEz. . _ Orr')r r rr r r rr(cSsg|] }|dqS)rr)rrrrrrwsz6test_cli_converters_conll_ner2json..r)r*r+r,r-r.r/r0cSsg|] }|dqS)rr)rrrrrrxsrzU-GPEzB-GPEzI-GPEzL-GPE)rrprintr r!r1)r"r#r$r2r%rrrr"test_cli_converters_conll_ner2json:sv   (r7c Cst}ddi}t||gdd\}}t|dks2t|dks>tdddgi}t||gdd\}}t|dksnt|dksztd}ttt||gddWdQRXd d i}ttt||gddWdQRXdd i}t||gdd\}}t|dkst|dks tdgi}t||gdd\}}t|dks:t|dksHtdd i} t|| gdd \}}t|dksvt|dkstddi} t|| gdd\}}t|dkst|dkstdS)Ntextz Some textr r'rrZSomedinvalidzDoes not matterr4zThis text is not long enoughz4This text contains way too much tokens for this testr5)rrr r!pytestZraises TypeError ValueError) ZnlpZvalid_jsonl_textZdocsZ skip_countZvalid_jsonl_tokensZinvalid_jsonl_typeZinvalid_jsonl_keyZempty_jsonl_textZempty_jsonl_tokensZtoo_short_jsonlZtoo_long_jsonlrrrtest_pretrain_make_docs|s>     r?) __future__rr<Z spacy.lang.enrZspacy.cli.convertersrrrZspacy.cli.pretrainrr&r3r7r?rrrrs   B