B
    .(b,                 @   sf   d dl mZ d dlZd dlmZ d dlmZ d dlmZ ej	
dddd	gfd
ddgfgdd ZdS )    )unicode_literalsN)Language)TOKENIZER_INFIXES)ALPHAztext,expected_tokenszl'avionzl'Zavionzj'aizj'Zaic                s\   dj td G  fdddt}|j }|| }t|dksBtdd |D |ksXtdS )	z?Allow zero-width 'infix' token during the tokenization process.z(?<=[{a}]\')(?=[{a}]))ac                   s"   e Zd ZG  fdddejZdS )z!test_issue768.<locals>.FrenchTestc                   s   e Zd Ze g ZdS )z*test_issue768.<locals>.FrenchTest.DefaultsN)__name__
__module____qualname__r   Zinfixes )SPLIT_INFIXr
   [/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/lang/fr/test_prefix_suffix_infix.pyDefaults   s   r   N)r   r   r	   r   r   r
   )r   r
   r   
FrenchTest   s   r      c             S   s   g | ]
}|j qS r
   )text).0tr
   r
   r   
<listcomp>   s    z!test_issue768.<locals>.<listcomp>N)formatr   r   r   Zcreate_tokenizerlenAssertionError)r   Zexpected_tokensr   Zfr_tokenizer_w_infixtokensr
   )r   r   test_issue768
   s    
r   )
__future__r   ZpytestZspacy.languager   Zspacy.lang.punctuationr   Zspacy.lang.char_classesr   markZparametrizer   r
   r
   r
   r   <module>   s   