B
    .(bB                 @   s  d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	 ddl
mZ dd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zejd(d)d*id+d*id,d*id-ggd.d/ Zd0d1 Zd2d3 Z d4d5 Z!dS )6    )unicode_literalsN)LEMMA)Vocab)DocToken   )get_docc          	   C   s   d}dddd}| |}t |dks(t| 2}|j|dd |d	 |j|dd |d	 W d Q R X t |d
ksxt|d jdkst|d jdkst|d jdkst|d jdkst|d jdkst|d jdkstd S )Nz-WKRO played songs by the beach boys all nightNAMEDr   TYPE)taglemmaent_type	         )attrs   zthe beach boyszthe beach boys    z	all night)lenAssertionError
retokenizemergetextZtext_with_wstag_)en_tokenizerr   r   docretokenizer r   T/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/doc/test_retokenize_merge.pytest_doc_retokenize_merge   s    
 r   c          	   C   s   d}dddd}| |}t |dks(t| }|j|dd |d	 W d
Q R X xR|D ]J}|j|jjk r|t|jjkstqX|j|jjkrX|t|jjksXtqXW d
S )z3Test that attachments work correctly after merging.z-WKRO played songs by the beach boys all nightr	   r   r
   )r   r   r   r   r   r   )r   N)	r   r   r   r   iheadlistZleftsZrights)r   r   r   r   r   wordr   r   r   "test_doc_retokenize_merge_children   s    
 
r$   c          	   C   s\   d}| |}|  >}|j|dd dddd |j|dd	 dddd W d Q R X d S )
Nz through North and South Carolina   r    ORG)r   r   )r      r   )r   r   )r   r   r   r   r   r   r   test_doc_retokenize_merge_hang,   s
    
r)   c          	   C   sT   | d}|  }||dd  W d Q R X t|dks>t|d jdksPtd S )Nz-WKRO played songs by the beach boys all nightr   r   zthe beach boys)r   r   r   r   r   )r   r   r   r   r   r   test_doc_retokenize_retokenizer4   s
    
r*   c          	   C   s   | d}t dd|jjd i}| }|j|dd |d W d Q R X t|dksVt|d jdksht|d jdkszt|d j	dkstd S )	Nz-WKRO played songs by the beach boys all nightboysZENT_TYPEr'   r   r   )r   zthe beach boys)
r   vocabstringsr   r   r   r   r   lemma_	ent_type_)r   r   r   r   r   r   r   %test_doc_retokenize_retokenizer_attrs<   s    
 r0   c          	   C   s   | d}t dd |D rt| "}|j|dd dddd	 W d
Q R X |d jdks`t|d jdksrt|d jstt|jdgd}|d jstd
S )z8Test that lexical attributes can be changed (see #2390).zWKRO played beach boys songsc             s   s   | ]}|j V  qd S )N)is_stop).0tokenr   r   r   	<genexpr>K   s    z0test_doc_retokenize_lex_attrs.<locals>.<genexpr>r   r   r+   T)r   ZIS_STOP)r   Nz
beach boys)wordsr   )	anyr   r   r   r   r.   r1   r   r,   )r   r   r   Znew_docr   r   r   test_doc_retokenize_lex_attrsH   s    
&r7   c          	   C   s   d}ddddg}| |}t |jdd |D |d}t|dksBt|d jjd	ksVt|d jjd
ksjt| (}dddd}|j|dd |d W d Q R X t|dkst|d jdkst|d jjd
kst|d jdkstd S )NzLos Angeles start.r(   r   c             S   s   g | ]
}|j qS r   )r   )r2   tr   r   r   
<listcomp>Y   s    z:test_doc_retokenize_spans_merge_tokens.<locals>.<listcomp>)r5   headsr   ZAngelesstartNNPzLos AngelesGPE)r   r   r   r   )r   r%   )	r   r,   r   r   r!   r   r   r   r/   )r   r   r;   tokensr   r   r   r   r   r   &test_doc_retokenize_spans_merge_tokensU   s    
 r@   c          	   C   sv  d}ddddg}| |}t |jdd |D ddd	d
gddddg|d}t|dksVt|d jdksht|d jdkszt|d jdkst| }||dd  W d Q R X t|dkst|d jdkst|d jdkst|d jdkst|d j	dkstt |jdd |D ddd	d
gddddg|d}t|dksLt|d jdks`t|d jdkstt|d jdkst| *}||dd  ||dd  W d Q R X t|dkst|d jdkst|d jdkst|d jdkst|d j	dks"t|d jdks6t|d jd	ksJt|d jdks^t|d j	dksrtd S )NzThe players start.r(   r   r8   c             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:   l   s    zHtest_doc_retokenize_spans_merge_tokens_default_attrs.<locals>.<listcomp>DTNNVBZ.ZDETZNOUNZVERBZPUNCT)r5   tagsposr;   r   ZTher   r%   zThe playersc             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:   ~   s    zstart .)
r   r,   r   r   r   r   Zpos_r   r   r.   )r   r   r;   r?   r   r   r   r   r   4test_doc_retokenize_spans_merge_tokens_default_attrsf   sT    





rG   c          	   C   s  d}ddddddddg}| |}t |jdd	 |D |d
}t|dksJt| .}|d jddd}|j|dd |d W d Q R X t|dkst|d jjdkst|d jjdkst|d jjdkst|d jjdkst|d jjdkst|d jjdkstd S )Nz"I found a pilates class near work.r(   r   r   r8   ic             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:      s    z9test_doc_retokenize_spans_merge_heads.<locals>.<listcomp>)r5   r;      r   zpilates classO)r   r   r   r%   r   )r   r   )r(   r%   )	r   r,   r   r   r   r   r   r!   r    )r   r   r;   r?   r   r   r   r   r   r   %test_doc_retokenize_spans_merge_heads   s    
 rK   c          
   C   sv   d}| |}t tV | B}|j|dd ddddd |j|dd	 ddddd W d Q R X W d Q R X d S )
NzLos Angeles start.r   r   r=   zLos Angelesr>   )r   r   r   )r   r(   )pytestraises
ValueErrorr   r   )r   r   r   r   r   r   r   ,test_doc_retokenize_spans_merge_non_disjoint   s    


rO   c             C   s  d}ddddddddg}| |}t |jdd |D |d	}|d
 jjdksNt| (}dddd}|j|dd |d W d Q R X |d jjdkstd}ddddddd
ddddddddddg}| |}t |jdd |D |d	}| 6}x.|jD ]$}|j|j	|jd}|j||d qW W d Q R X d}ddddddddd
ddddddg}| |}t |jdd |D |d	}| "}x|jD ]}|| qW W d Q R X d S )Nz.displaCy is a parse tool built with Javascriptr(   r   r   rH   r8   c             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:      s    z6test_doc_retokenize_span_np_merges.<locals>.<listcomp>)r5   r;   r   ZNPZtoolrJ   )r   r   r   r   )r   zmdisplaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript.rI   r%   iic             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:      s    zFOne test with entities like New York City so the ents list is not void   c             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:      s    )
r   r,   r!   r    r   r   r   entsZlabel_r.   )r   r   r;   r?   r   r   r   entr   r   r   "test_doc_retokenize_span_np_merges   s0    
 &
"
rT   c             C   s   d}ddddddddddddd	ddd
dg}dddddddddddddddddg}dddg}| |}t |jdd |D |||d}t|dkst| D}x<|jD ]2}tdd |D }	|jj|	d}
|j	||
d  qW W d Q R X t|d!kstd S )"NzPStewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.
r(   r   r   r8   rP   rH   iir=   rC   rA   ZVBZRPrB   ZWPINCCrD   SP)r   r   PERSON)
   rQ   r>   )      rY   c             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:      s    z:test_doc_retokenize_spans_entity_merge.<locals>.<listcomp>)r5   r;   rE   rR      c             s   s   | ]}|j V  qd S )N)r/   )r2   wr   r   r   r4      s    z9test_doc_retokenize_spans_entity_merge.<locals>.<genexpr>)r   r   )r   r\   )
r   r,   r   r   r   rR   maxrootr.   r   )r   r   r;   rE   rR   r?   r   r   rS   r   r   r   r   r   &test_doc_retokenize_spans_entity_merge   s    &&

ra   c          	   C   s>  dddddg}t t |d}|jjddd	f|jjd
d	dfg|_|d jdksXt|d jdksjt|d jdks|t|d	 jdkst| }|	|dd  W d Q R X t
|t
|d kst|d jdkst|d jdkstdddddg}t t |d}| <}ddd}|j	|dd	 |d |j	|d	d |d W d Q R X |d jdksjt|d jdks~tdddddddddg	}t t |d}|jjdd	df|jjdddfg|_|d	 jdkst|d jdkst|d jdkst|d jdkst| <}|	|dd  |	|dd  |	|dd  W d Q R X t
|dksxt|d	 jdkst|d	 jdkst|d jdkst|d jdkstdddddddddg	}dddddddd d!g	}d"d#g}d$gt
| }| jd | jd | jd$ t| ||||d%}|dd j|d	 ks\t|dd j|d ksxt| <}|	|dd  |	|dd  |	|dd  W d Q R X t
|dkst|d jdkst|d jdkst|d	 jdkst|d	 jdks$t|d jdks8t|d jdksLtdddddddddg	}dddddddd d!g	}d"d&g}d$gt
| }t| ||||d%}| *}|	|d	d  |	|dd  W d Q R X t
|dkst|d	 jdkst|d	 jdkst|d jdks&t|d jdks:td S )'Nabcde)r5   zent-abcr   r%   zent-dr   Br(   Ir   )r   Zent_iob)r   r   fghr    zent-dezent-fgr   r   r   r8   rH   rU   ii)r%   r   zent-de)r   r   zent-fgdep)r5   r;   depsrR   )r   r   zent-de)r   r   r,   r-   addrR   Zent_iob_r   r   r   r   r/   r   r`   )en_vocabr5   r   r   r   r;   rR   rn   r   r   r   *test_doc_retokenize_spans_entity_merge_iob   s    


 


rq   c             C   s
  d}dddddddddddddddd	d
g}dddddddddddddddddg}| |}t |jdd |D ||d}t|j\}}t|}t|}	| <}
ddd}|
j|dd |d |
j|d	d  |d W d Q R X t||d kstt||	d kstd S )NzOStewart Lee is a stand up comedian. He lives in England and loves Joe Pasquale.r(   r   r   r8   rU   rl   rH   rP   icompoundnsubjROOTdetamodprtattrpunctpreppobjccconjdobjc             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:   J  s    zItest_doc_retokenize_spans_sentence_update_after_merge.<locals>.<listcomp>)r5   r;   rn   none)r   r   )r   )r   r,   r"   sentsr   r   r   r   )r   r   r;   rn   r?   r   sent1Zsent2init_lenZ	init_len2r   r   r   r   r   5test_doc_retokenize_spans_sentence_update_after_mergeA  s     &


 r   c       
      C   s   d}ddddddddddddd	ddg}d
ddddddddddddd
dg}| |}t |jdd |D ||d}t|jd }tt|jj}| &}ddd}	|j|dd |	d W d Q R X tt|jj|d kst	d S )NzNStewart Lee is a stand up comedian who lives in England and loves Joe Pasqualer(   r   r   r8   rU   rP   rH   irr   rs   rt   ru   rv   rw   rx   Zrelclrz   r{   r|   r}   r~   c             S   s   g | ]
}|j qS r   )r   )r2   r9   r   r   r   r:   _  s    z@test_doc_retokenize_spans_subtree_size_check.<locals>.<listcomp>)r5   r;   rn   r   )r   r   )r   )
r   r,   r"   r   r   r`   Zsubtreer   r   r   )
r   r   r;   rn   r?   r   r   r   r   r   r   r   r   ,test_doc_retokenize_spans_subtree_size_checkV  s    "

 r   c          	   C   sj  t jdddd t jdddd t| ddd	gd
}| ,}ddddd}|j|dd |d W d Q R X |d jdks~t|d jjdkst|d jj	dkstt| ddd	d	gd
}| F}|j|dd ddddid |j|dd dd ddid W d Q R X |d jjdks$t|d jj	dks:t|d jjd ksPt|d jj	dksftd S )Nrb   FT)defaultforcerc   Znothinghelloworld!)r5   zhello world1)rb   rc   )r   _r   r   )r   r   r   2r(   )
r   set_extensionr   r   r   r.   r   r   rb   rc   )rp   r   r   r   r   r   r   )test_doc_retokenize_merge_extension_attrsh  s"    
 
 *r   underscore_attrsrb   xrc   rd   r(   c          
   C   s   t jddd dd t jddd dd t| d	d
dgd}d|i}tt0 | }|j|dd |d W d Q R X W d Q R X d S )Nrb   c             S   s   | S )Nr   )r   r   r   r   <lambda>      zCtest_doc_retokenize_merge_extension_attrs_invalid.<locals>.<lambda>T)getterr   rc   c             S   s   | S )Nr   )r   r   r   r   r     r   )methodr   r   r   r   )r5   r   r   r   )r   )r   r   r   rL   rM   rN   r   r   )rp   r   r   r   r   r   r   r   1test_doc_retokenize_merge_extension_attrs_invalid~  s    
r   c          	   C   s  t | dddgd}tdd |D r(t| "}|j|dd d	d
dd W dQ R X |d jd	ksjt|d jsxtt | ddddgd}tdd |D rttdd |D rt| :}|j|dd dd
id |j|dd dd
id W dQ R X |d jst|d js t|d jr0t|d jr@tt | ddddgd}|d jdksht|  }|j|dd ddid W dQ R X |d jdkst| d jdkstdS )a  Test that retokenization also sets attributes on the lexeme if they're
    lexical attributes. For example, if a user sets IS_STOP, it should mean that
    "all tokens with that lexeme" are marked as a stop word, so the ambiguity
    here is acceptable. Also see #2390.
    r   r   r   )r5   c             s   s   | ]}|j V  qd S )N)r1   )r2   r9   r   r   r   r4     s    z7test_doc_retokenizer_merge_lex_attrs.<locals>.<genexpr>r   r   zhello worldT)r   r1   )r   NZeinsZzweic             s   s   | ]}|j V  qd S )N)like_num)r2   r9   r   r   r   r4     s    c             s   s   | ]}|j V  qd S )N)r1   )r2   r9   r   r   r   r4     s    r   r   r1   r(   Znormr   )	r   r6   r   r   r   r.   r1   r   Znorm_)rp   r   r   r   r   r   $test_doc_retokenizer_merge_lex_attrs  s,    
&
$
$r   c          	   C   sp   t | dddgd}| *}||dd  ||dd  W dQ R X t|dksZt|d jdksltdS )	zsTest that the retokenizer automatically skips duplicate spans instead
    of complaining about overlaps. See #3687.r   r   r   )r5   r   r   Nzhello world)r   r   r   r   r   r   )rp   r   r   r   r   r   test_retokenize_skip_duplicates  s    
r   c          
   C   sR   t | dddgd}tt, | }||dd  W d Q R X W d Q R X d S )Nr   r   r   )r5   r(   )r   rL   rM   rN   r   r   )rp   r   r   r   r   r   $test_retokenize_disallow_zero_length  s    
r   )"
__future__r   rL   Zspacy.attrsr   Zspacy.vocabr   Zspacy.tokensr   r   utilr   r   r$   r)   r*   r0   r7   r@   rG   rK   rO   rT   ra   rq   r   r   r   markZparametrizer   r   r   r   r   r   r   r   <module>   s2   /Y,!