B
    .(b                 @   sP   d dl mZ d dlmZ d dlmZ d dlmZ ddlmZ dd Z	d	d
 Z
dS )    )unicode_literals)English)	Tokenizer)util   )make_tempdirc        
   	   C   s   d} t  }|| }dd |D }t| || }dd |D }t }|| t|}W d Q R X || }dd |D }	||	kstd S )NzTest c.c             S   s   g | ]
}|j qS  )text).0tokenr   r   T/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/regression/test_issue4190.py
<listcomp>   s    z"test_issue4190.<locals>.<listcomp>c             S   s   g | ]
}|j qS r   )r	   )r
   r   r   r   r   r      s    c             S   s   g | ]
}|j qS r   )r	   )r
   r   r   r   r   r      s    )r   customize_tokenizerr   Zto_diskr   Z
load_modelAssertionError)
test_stringZnlp_1Zdoc_1aZ	result_1aZdoc_1bZ	result_1bZ	model_dirZnlp_2Zdoc_2Zresult_2r   r   r   test_issue4190   s    
r   c             C   sn   t | jj}t | jj}t | jj}dd t| jj	
 D }t| j||j|j|j| jjd}|| _d S )Nc             S   s.   i | ]&\}}t |d kr$|d dks||qS )r      .)len)r
   kvr   r   r   
<dictcomp>$   s   z'customize_tokenizer.<locals>.<dictcomp>)Zprefix_searchZsuffix_searchZinfix_finditertoken_match)r   Zcompile_prefix_regexZDefaultsprefixesZcompile_suffix_regexsuffixesZcompile_infix_regexZinfixesdictZtokenizer_exceptionsitemsr   Zvocabsearchfinditer	tokenizerr   )ZnlpZ	prefix_reZ	suffix_reZinfix_re
exceptionsZnew_tokenizerr   r   r   r      s    r   N)
__future__r   Zspacy.lang.enr   Zspacy.tokenizerr   Zspacyr   r   r   r   r   r   r   r   <module>   s   