B
    .(bK                 @   s\  d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 ddl
mZ g g fdd	d
gdddgfgZdd	d
gdfgZejjejddgdd Zejdedd Zejdedd Zejdedd Zejdedd Zejdedd Zejdedd Zejded d! Zejje	d"d#ejded$d% ZdS )&    )unicode_literalsN)Vocab)StringStore)
is_python2   )make_tempdirZratsZareZcuteilikeZHellotextZratc             C   sR   | j |}| jdgd}t |}|j | |ks8t|jdgd|ksNtd S )NZlookups)exclude)stringsaddto_bytesr   
from_bytesAssertionError)Zen_vocabr
   Z	text_hashZvocab_bytesZ	new_vocab r   a/home/dcms/DCMS/lib/python3.7/site-packages/spacy/tests/serialize/test_serialize_vocab_strings.pytest_serialize_vocab   s
    r   zstrings1,strings2c             C   s   t | d}t |d}| }| }| |kr:||ksFtn||ksFt||}| |ks`tt  |}| |ks|tt|jt| d ksttdd |jD t| dg kstd S )N)r      c             S   s   g | ]}|qS r   r   ).0sr   r   r   
<listcomp>*   s    z8test_serialize_vocab_roundtrip_bytes.<locals>.<listcomp>_SP)r   r   r   r   lenr   sorted)strings1strings2vocab1vocab2Zvocab1_bZvocab2_bZ
new_vocab1r   r   r   $test_serialize_vocab_roundtrip_bytes   s    


r   c       	   	   C   s   t | d}t |d}t }|d }|d }|| || t  |}t  |}| dd |jD kspt|dd |jD kst| |krdd |jD dd |jD kstn$d	d |jD d
d |jD kstW d Q R X d S )N)r   r   r   c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   9   s    z7test_serialize_vocab_roundtrip_disk.<locals>.<listcomp>c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   :   s    c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   <   s    c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   =   s    c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   @   s    c             S   s   g | ]}|d kr|qS )r   r   )r   r   r   r   r   r   A   s    )r   r   to_disk	from_diskr   r   )	r   r   r   r   d
file_path1
file_path2Zvocab1_dZvocab2_dr   r   r   #test_serialize_vocab_roundtrip_disk-   s     



r%   zstrings,lex_attrc             C   sr   t | d}t  }||| d  _|| d  j|ks4t|| d  j|ksJt|| }|| d  j|ksntd S )N)r   r   )r   norm_r   r   r   )r   lex_attrr   r   r   r   r   $test_serialize_vocab_lex_attrs_bytesE   s    
r(   c             C   s6   t | d}||  t|jt| d ks2td S )N)r   r   )r   r   r   r   r   r   )r   r'   vocabr   r   r   #test_deserialize_vocab_seen_entriesP   s    
r*   c          	   C   s   t | d}t  }||| d  _|| d  j|ks4t|| d  j|ksJtt "}|d }|| ||}W d Q R X || d  j|kstd S )N)r   r   r)   )r   r&   r   r   r    r!   )r   r'   r   r   r"   	file_pathr   r   r   #test_serialize_vocab_lex_attrs_diskX   s    

r,   c             C   s   t | d}t |d}| }| }| |kr:||ksFtn||ksFt||}| |ks`tt  |}| |ks|tt|| kstd S )N)r   )r   r   r   r   list)r   r   sstore1sstore2Z	sstore1_bZ	sstore2_bZnew_sstore1r   r   r   *test_serialize_stringstore_roundtrip_bytesf   s    


r0   c       	   	   C   s   t | d}t |d}t }|d }|d }|| || t  |}t  |}t|t|ksltt|t|kst| |krt|t|kstnt|t|kstW d Q R X d S )N)r   r   r   )r   r   r    r!   r-   r   )	r   r   r.   r/   r"   r#   r$   Z	sstore1_dZ	sstore2_dr   r   r   )test_serialize_stringstore_roundtrip_diskw   s    



r1   z+Dict order? Not sure if worth investigating)reasonc             C   sD   t | d}||| d  _t|}t|}| | ks@td S )N)r   r   )r   r&   pickledumpsloadsr   r   )r   r'   r)   Zvocab_pickledZvocab_unpickledr   r   r   test_pickle_vocab   s
    


r6   )
__future__r   Zpytestr3   Zspacy.vocabr   Zspacy.stringsr   Zspacy.compatr   utilr   Ztest_stringsZtest_strings_attrsmarkZxfailZparametrizer   r   r%   r(   r*   r,   r0   r1   Zskipifr6   r   r   r   r   <module>   s&   	