B .(b @sddlmZddlZddlmZddlmZmZddlm Z ddZ d d Z d d Z d dZ ddZddZddZejdddiigddiigddiigddiddigdddddigddigddZddZdd Zd!d"ZdS)#)unicode_literalsN)Vocab)DocToken)get_docc Csdddg}dddg}t|||d}t|dks2ttt|dksFt|djjdksZt|djjdksnt|H}|j|dd d g|ddf|dgd gd d d gd gd ddWdQRXt|dkst|djd kst|djjd kst|djdkst|djdks"t|djd ks6t|djjdksLt|d jdks`t|d jjdksvt|djdkst|djjdksttt|dkstdS)N LosAngelesstart.r)wordsheadsLosAngelesZNNPrZGPE)taglemmaZent_type)attrs) rlenAssertionErrorstrheadtext retokenizesplitidx)en_vocabr r doc retokenizerr!T/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/doc/test_retokenize_split.pytest_doc_retokenize_split s6   r#c Cst|dddgd}|jjd}|jjd}|8}|j|ddd g|dd f|d gd ||gid WdQRX|dj|kst|d j|kstdS) Nrr r )r amodsubjectrrrr dep)r)rvocabstringsaddrrr&r)rrZdep1Zdep2r r!r!r"&test_doc_retokenize_split_dependencies,s r*c Cst|dddgd}tt6|"}||dddg|dgWdQRXWdQRXttB|.}||dddg|d|d|dgWdQRXWdQRXdS) Nrr r )r rrrr )rpytestraises ValueErrorrr)rrr r!r!r"%test_doc_retokenize_split_heads_error;s  0  r.c Csdddg}tt|d}|jjdddfg|_|djdksBt|d jd ksTt|8}| |dd d d g|dd f|ddf|d gWdQRX|djdkst|d jd kst|djd kst|djd kstdS)Nabcde)r zent-abcdrrBr Iabcr) rrr'r(r)ZentsZent_iob_rrr)r rr r!r!r"*test_doc_retokenize_spans_entity_split_iobHs  <r7c CsJddddddddd d d d d ddg}dddddddddddddddg}dddddddddddd d!d"dg}t||||d#}t|j\}}t|}t|}|j} | j|dd$d%g|ddf|dgd&d"dgid'| j|d(d)d*g|d(df|d+gd&d"d,gid'WdQRXt|j\}}t||dks0tt||dksFtdS)-NZ StewartLeeisr4ZstandZupZcomedianr ZHeZlivesinZEnglandandZlovesZ JoePasqualer rrZnsubjROOTZdetr$ZprtattrpunctprepZpobjccZconjZcompound)r r depsZStewartZLeer&)r ZJoeZPasquale Zdobj)rlistZsentsrrrr) rr r rErZsent1Zsent2Zinit_lenZ init_len2r r!r!r"5test_doc_retokenize_spans_sentence_update_after_splitWs." rIc Csjt|dddgd}ttD|0}||dddg|ddf|ddfgWdQRXWdQRXdS) alTest that the regular retokenizer.split raises an error if the orths don't match the original token text. There might still be a method that allows this, but for the default use cases, merging and splitting should always conform with spaCy's non-destructive tokenization policy. Otherwise, it can lead to very confusing and unexpected results. rr r )r rLAN)rr+r,r-rr)rrr r!r!r"(test_doc_retokenize_split_orths_mismatchus  rLc Cs tjddddtjddddt|ddgd }|T}|d d f|d g}dd d ddig}ddg|d}|j|d ddg||dWdQRX|d jdkst|d jjdkst|d jj d kst|d jdkst|d jjdkst|d jj dkstdS)Nr4FT)defaultforcer5Znothingrr )r rr 1)r4r52ZlosZangeles)r_rr)r) r set_extensionrrrZlemma_rrQr4r5)rrr r Z underscorerr!r!r")test_doc_retokenize_split_extension_attrss $rSunderscore_attrsr4xr5r6)r4rUc Cstjddddtjdddddtjd d ddd t|d d gd}d|i}ttH|4}|ddf|dg}|j|dddg||dWdQRXWdQRXdS)NrUFT)rMrNr4cSs|S)Nr!)rUr!r!r"zCtest_doc_retokenize_split_extension_attrs_invalid..)getterrNr5cSs|S)Nr!)rUr!r!r"rVrW)methodrNrr )r rQrr rr)r)rrRrr+r,r-rr)rrTrrr r r!r!r"1test_doc_retokenize_split_extension_attrs_invalids   rZc Cst|dgddjrtt|dgddjr0tt|ddgd}|djrNt|@}ddd gi}|dd f|d g}|j|dddg||d Wd QRX|djst|d jrtd S) aTest that retokenization also sets attributes on the lexeme if they're lexical attributes. For example, if a user sets IS_STOP, it should mean that "all tokens with that lexeme" are marked as a stop word, so the ambiguity here is acceptable. Also see #2390. r)r rrrr is_stopTFr )rN)rr[rrr)rrr rr r!r!r"$test_doc_retokenizer_split_lex_attrss  $r\c Csd}t||ddd}|<}|d}|dfgt|}|j||jt|j|dWdQRXt||d}|<}|d}|dfgt|}|j||jt|j|dWdQRXdS)zB#4604: realloc correctly when new tokens outnumber original tokenszOHyperglycemic adverse events following antipsychotic drug administration in theNr;)r r)r )rrrrirHr)rrrr tokenr r!r!r"test_doc_retokenizer_reallocs & r_c sd}t||d}d|d_|d|2}|jdddd d gfd d tdDd WdQRX|djdksxt|djdkst|djd kst|djd kstdS)z#6060: reset norm in splitz6The quick brownfoxjumpsoverthe lazy dog w/ white spots)r withrZbrownZfoxZjumpsZoverZthecsg|] }|fqSr!r!).0r)r^r!r" sz3test_doc_retokenizer_split_norm..)r N zw/)rrZnorm_rrangerr)rrrr r!)r^r"test_doc_retokenizer_split_norms  6rf) __future__rr+Z spacy.vocabrZ spacy.tokensrrutilrr#r*r.r7rIrLrSmarkZ parametrizerZr\r_rfr!r!r!r"s,   !