B
    `nf                 @   s   d dl Z d dlZd dlZd dlmZmZmZmZ d dl	m
Z d dlmZ ejdd Zejdd Zejdd	 Zejd
d Zejdd ZG dd dZG dd dZdS )    N)	DataFrameIndexSeriesjson_normalize)nested_to_recordc               C   sl   ddddddddgdd	d
dddddgdgddddddgddddddddgdgdgS )NUSA
CaliforniazSan Franciscoi90  )namepopzLos Angelesi:0  )r	   citiesOhioColumbusi  	Clevelandi  )countrystatesGermanyBayernMunichi;0  zNordrhein-WestfalenDuesseldorfi  Koelni   r   r   r   R/home/dcms/DCMS/lib/python3.7/site-packages/pandas/tests/io/json/test_normalize.pydeep_nested   s    r   c               C   sL   dddddddddgdd	id
ddddddddgddidddgS )NDadei90  )r	   
populationBrowardi@  z
Palm Beachi`  governorz
Rick ScottFLFlorida)countiesinfo	shortnamestateSummiti  Cuyahogai9  zJohn KasichOHr   r   r   r   r   r   
state_data3   s    

r&   c               C   s   dd idddddddgS )	Nr    z
11/08/1993z
26/05/2012)
created_atlast_updatedJaneDoe)first	last_name)r    author_namer   r   r   r   r   author_missing_dataL   s    r.   c               C   s.   dddddddgdd	d
dddddgigS )NAliceiZ%  z
Morris St.	Massillonr%   if  )numberstreetcityr"   zip)r	   	addressesr5   i!  z
Spring St.ElizabethtonTNi  r   r   r   r   r   missing_metadataW   s    r8   c               C   s"   ddiddddddd	id
gS )z@
    input data to test json_normalize with max_level param
    NameUser001z	Some textID001Name001)Idr9   )	TextField	UserFieldab)	CreatedByLookupImager   r   r   r   r   max_level_test_input_datat   s    rE   c               @   s6  e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zejdddd d!d"d#d$id%d&id'dd d!d"d#d$id%d&id'gfd(dd d!d#d$id%d&id)dd d!d#d$id%d&id)gfgd*d+ Zd,d- Zd.d/ Zejd0d1d2d3d4d5gd6d7 Zd8d9 Zd:S );TestJSONNormalizec             C   sL   dddddddddd	d
dddddg}t |}t|}t|| d S )N         )r@   rA   c               	   
         )r   r   tmassert_frame_equal)selfrecsresultexpectedr   r   r   test_simple_records   s    


z%TestJSONNormalize.test_simple_recordsc             C   s   t |d d}t|d d }t|| t |d}g }x|D ]}||d  q>W t|}t|| t |ddd}tddgddg|d< t|| d S )	Nr   r   r"   )metar   r   rI   rH   )r   r   rT   rU   extendnparrayrepeat)rV   r&   rX   rY   recr   r   r   test_simple_normalize   s    

z'TestJSONNormalize.test_simple_normalizec             C   s   t g }t }t|| d S )N)r   r   rT   rU   )rV   rX   rY   r   r   r   test_empty_array   s    z"TestJSONNormalize.test_empty_arrayc             C   s   t ddddi}tddggddgd}t||| t ddddidd	}tddggd
dgd}t||| t ddddidd	}tddggddgd}t||| t |ddgdddggdd}tddddg }|j |st	d S )NArG   rH   )rc   BzA.AzA.B)columns_)sepZA_AZA_Bu   σu   AσAu   AσBr   r   r   r	   )r[   rg   r
   Zstates_name)
r   r   rT   rU   Zreindex_liker   Zsort_valuesre   equalsAssertionError)rV   r   rX   rY   r   r   r   $test_simple_normalize_with_separator   s     
z6TestJSONNormalize.test_simple_normalize_with_separatorc             C   s<   t dddgiddd}tdgdggdgd}t|| d S )Nrc   rG   rH   zPrefix.)record_prefixzPrefix.0)re   )r   r   rT   rU   )rV   rX   rY   r   r   r   test_value_array_record_prefix   s    z0TestJSONNormalize.test_value_array_record_prefixc             C   sj   dddddddddd	dgd
d}t |ddgd}tddgddgdd	ggddgd}t|| d S )Nr   z
Rick Scottr   i90  )r	   r   r   i@  z
Palm Beachi`  )r   r   )r"   r    r    r   )record_pathr	   r   )re   )r   r   rT   rU   )rV   datarX   rY   r   r   r   test_nested_object_record_path   s    z0TestJSONNormalize.test_nested_object_record_pathc          
   C   s   t |ddgdddggd}dgd dgd	  d
d
dddddgdddddddgdddddddgd}t||jd}t|| d S )Nr   r   r   r	   )r[   r   rK   r   rI   r   r   r   zNordrhein-WestfalenzSan FranciscozLos Angelesr   r   r   r   r   i90  i:0  i  i  i;0  i  i  )r   zstates.namer	   r
   )re   )r   r   re   rT   rU   )rV   r   rX   ex_datarY   r   r   r   test_more_deeply_nested   s(    z)TestJSONNormalize.test_more_deeply_nestedc          	   C   s   ddddiddddd	dd
ddgdddddiddddddgdg}t |dddddgg}ddd
ddgdgd dgd  dddddgdgd dgd  dd	dddgd}t||jd}t|| d S )Nr   r   r   z
Rick Scottr   i90  )r	   r   r   i@  z
Palm Beachi`  )r"   r!   r    r   r   r%   zJohn Kasichr#   i  r$   i9  r   r"   r!   r    rI   rH   )r	   r"   r!   zinfo.governorr   )re   )r   r   re   rT   rU   )rV   rn   rX   rp   rY   r   r   r   test_shallow_nested   s(    z%TestJSONNormalize.test_shallow_nestedc             C   s   t |dgddddggdd}dd	d
ddgdddddgdgd dgd  dgd dgd  dgd dgd  d}t|}t|| d S )Nr   r"   r!   r    r   ignore)rn   rm   r[   errorsr   r   z
Palm Beachr#   r$   i90  i@  i`  i  i9  r   rI   r   rH   r   r%   z
Rick ScottzJohn Kasich)r	   r   r"   r!   zinfo.governor)r   r   rT   rU   )rV   r&   rX   rp   rY   r   r   r   -test_nested_meta_path_with_nested_record_path  s    z?TestJSONNormalize.test_nested_meta_path_with_nested_record_pathc          	   C   s   ddddddddgdg}d	}t jt|d
 t|dddgd W d Q R X t|dddgdd}xdD ]}||kshtqhW d S )Nhellothere	somethingelse)foobar
something2else2)rz   r{   rn   z?Conflicting metadata name (foo|bar), need distinguishing prefix)matchrn   rz   r{   )r[   r[   )r[   meta_prefix)metafoometabarrz   r{   )pytestraises
ValueErrorr   ri   )rV   rn   msgrX   valr   r   r   test_meta_name_conflict3  s    
z)TestJSONNormalize.test_meta_name_conflictc             C   sd   ddddddddgdg}d	d
g}t |d|dd}|d	d
gksFtxdD ]}||ksLtqLW d S )Nrv   rw   rx   ry   )rz   r{   r|   r}   )rz   r{   rn   rz   r{   rn   r[   )r[   r   )r   r   rz   r{   )r   ri   )rV   rn   COLUMNSrX   r   r   r   r    test_meta_parameter_not_modifiedH  s    
z2TestJSONNormalize.test_meta_parameter_not_modifiedc             C   s   t |d d}t|d d }t|| t |dddd}g }x|D ]}||d  qDW t|}|jdd d}td	d
gddg|d< t|| d S )Nr   r   r"   county_)r[   rk   c             S   s   d|  S )Nr   r   )xr   r   r   <lambda>i      z6TestJSONNormalize.test_record_prefix.<locals>.<lambda>)re   r   r   rI   rH   )	r   r   rT   rU   r\   renamer]   r^   r_   )rV   r&   rX   rY   r`   r   r   r   test_record_prefix\  s    
z$TestJSONNormalize.test_record_prefixc             C   sR   d d}d dddgdddgdd	d
gi}t|}tt|}t|| d S )NsK   [{"Ünicøde":0,"sub":{"A":1, "B":2}},{"Ünicøde":1,"sub":{"A":3, "B":4}}]utf8s	   Ünicøder   rG   zsub.ArI   zsub.BrH   rK   )decoder   r   jsonloadsrT   rU   )rV   ZtestjsonZtestdatarY   rX   r   r   r   test_non_ascii_keyn  s    z$TestJSONNormalize.test_non_ascii_keyc             C   sJ   t |}tjtjtjtjtjdd dddddg}t|}t|| d S )N)r    zinfo.created_atzinfo.last_updatedzauthor_name.firstzauthor_name.last_namez
11/08/1993z
26/05/2012r)   r*   )r   r]   nanr   rT   rU   )rV   r.   rX   rp   rY   r   r   r   test_missing_field~  s    
z$TestJSONNormalize.test_missing_fieldzmax_level,expectedr   z	Some textr;   r<   )r=   r9   r9   r:   r@   rA   )r>   r?   rB   rD   rG   )r>   zUserField.IdzUserField.NamerB   rD   c             C   s~   ddiddddddddddgdd	id
dddddgdg}t |dgdgdgg|d}t||jjd}t|| d S )Nr9   r:   z	Some textr;   r<   )r=   r9   )r>   r?   r@   rA   rx   ry   )rz   r{   r|   r}   )rB   rC   rD   tagsrC   rB   rD   )rm   r[   	max_level)rn   re   )r   r   re   valuesrT   assert_equal)rV   r   rY   
test_inputrX   Zexpected_dfr   r   r    test_max_level_with_records_path  s    -
z2TestJSONNormalize.test_max_level_with_records_pathc             C   s:   t dddiig}t ddddiigid}t|| d S )Nrc   rd   rG   dummy)r   rT   rU   )rV   Zdf1Zdf2r   r   r   !test_nested_flattening_consistent  s    z3TestJSONNormalize.test_nested_flattening_consistentc             C   sF   t d|ddddigdgdgd}tddidgd	}t|| d S )
NTexas)r"   r    r   irH   r    )rm   r   )index)r   r   rT   r   )rV   Znulls_fixturerX   rY   r   r   r   test_nonetype_record_path  s    
z+TestJSONNormalize.test_nonetype_record_pathvaluefalsetruez{}1z"text"c          	   C   s\   t |}d|d}d}| d| d| d}tjt|d t|g|gd W d Q R X d S )	Nr   )r"   r    r    z has non list value z
 for path z. Must be list or null.)r~   )rm   )r   r   r   r   	TypeErrorr   )rV   r   Zparsed_valuer   Z	test_pathr   r   r   r    test_non_list_record_path_errors  s    

z2TestJSONNormalize.test_non_list_record_path_errorsc             C   sL   d}t t|dgdgd}tdgdgtjdgtdd	}t|| d S )
Nz,[{"id": 99, "data": [{"one": 1, "two": 2}]}]rn   id)rm   r[   rG   rH   c   )Zdtype)ZoneZtwor   )	r   r   r   r   r]   r^   objectrT   rU   )rV   rn   rX   rY   r   r   r   test_meta_non_iterable  s
    z(TestJSONNormalize.test_meta_non_iterableN)__name__
__module____qualname__rZ   ra   rb   rj   rl   ro   rq   rr   ru   r   r   r   r   r   r   markparametrizer   r   r   r   r   r   r   r   r   rF      sP   $

& rF   c               @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
ejddddddddgfdddidddddddidgfdddddddd gfgd!d" Zd#d$ Zd%d& Zd'd( ZdS ))TestNestedToRecordc             C   s4   t dddt dddg}t|}|}||ks0td S )NrG   rH   )flat1Zflat2rI   rK   )dictr   ri   )rV   rW   rX   rY   r   r   r   test_flat_stays_flat  s    z'TestNestedToRecord.test_flat_stays_flatc             C   s8   t dt dddd}t|}dddd}||ks4td S )NrG   rH   )rJ   d)r   dict1)zdict1.czdict1.dr   )r   r   ri   )rV   rn   rX   rY   r   r   r   test_one_level_deep_flattens  s    z/TestNestedToRecord.test_one_level_deep_flattensc             C   sP   t dt dddt t dddddd}t|}ddddddd}||ksLtd S )NrG   rH   )rJ   r   )er   )r   r   nested)zdict1.czdict1.dr   znested.dz
nested.e.cz
nested.e.d)r   r   ri   )rV   rn   rX   rY   r   r   r   test_nested_flattens  s    &z'TestNestedToRecord.test_nested_flattensc          	   C   s2   d}t jt|d t|dddd W d Q R X d S )NzDTry running with errors='ignore' as key 'name' is not always present)r~   r5   r	   raise)rn   rm   r[   rt   )r   r   KeyErrorr   )rV   r8   r   r   r   r   test_json_normalize_errors(  s    z-TestNestedToRecord.test_json_normalize_errorsc             C   sn   t |dddd}ddddd	d
gdddddtjgg}ddddddg}ddddddg}t||d}t|| d S )Nr5   r	   rs   )rn   rm   r[   rt   iZ%  z
Morris St.r0   r%   if  r/   i!  z
Spring St.r6   r7   i  r3   r1   r"   r2   r4   )re   )r   r]   r   r   rT   rU   )rV   r8   rX   rp   re   rY   r   r   r   test_missing_meta6  s    z$TestNestedToRecord.test_missing_metac             C   sX   d ddddddddd	ddg}t |}d ddd
dd	dddg}||ksTtd S )NZSmithZ	Appleseed)r+   r,   )r    r-   z
11/08/1993z
26/05/2012)r'   r(   r)   r*   )r    zauthor_name.firstzauthor_name.last_name)zauthor_name.firstzauthor_name.last_namezinfo.created_atzinfo.last_updated)r   ri   )rV   rn   rX   rY   r   r   r   test_donot_drop_nonevaluesF  s    
z-TestNestedToRecord.test_donot_drop_nonevaluesc          
   C   sP   d ddd d d dddddiid}t |}d d d d dddd	}||ksLtd S )
Nr   r"   g    gH@g   @g   ;@)r   regionr   yz)r   z	town.info)r   location)r   zlocation.country.state.idz#location.country.state.town.info.idz'location.country.state.town.info.regionz"location.country.state.town.info.xz"location.country.state.town.info.yz"location.country.state.town.info.z)r   ri   )rV   rn   rX   rY   r   r   r   $test_nonetype_top_level_bottom_level`  s$    z7TestNestedToRecord.test_nonetype_top_level_bottom_levelc          	   C   sT   d d d d d dddddddd}t |}d d d d d dddd	}||ksPtd S )
Ng    gH@g   @g   ;@)r   r   r   r   )r   z	town.info)r   r"   )r   r   )r   r   )r   zlocation.idzlocation.country.idzlocation.country.state.idz'location.country.state.town.info.regionz"location.country.state.town.info.xz"location.country.state.town.info.yz"location.country.state.town.info.z)r   ri   )rV   rn   rX   rY   r   r   r   test_nonetype_multiple_levels  s$    z0TestNestedToRecord.test_nonetype_multiple_levelszmax_level, expectedNr:   z	Some textr;   r<   rA   )zCreatedBy.NamezLookup.TextFieldzLookup.UserField.IdzLookup.UserField.NamezImage.ar   r9   )r=   r9   )r>   r?   r@   )rB   rC   rD   rG   )zCreatedBy.NamezLookup.TextFieldzLookup.UserFieldzImage.ac             C   s   t ||d}||kstd S )N)r   )r   ri   )rV   r   rY   rE   outputr   r   r   test_with_max_level  s    +z&TestNestedToRecord.test_with_max_levelc          	   C   sh   d}dddddddd	d
ddidddidiig}dddd	d
ddg}t ||d}||ksdtd S )Nd   rB   userZLeoZThomson)Z	firstnameZLastNamefatherZ	Father001Z	Father002Z	Father003r9   Z	Father004)r	   r   )r9   r   )r	   Zfamily_tree)zCreatedBy.user.name.firstnamezCreatedBy.user.name.LastNamez&CreatedBy.user.family_tree.father.namez-CreatedBy.user.family_tree.father.father.Namez4CreatedBy.user.family_tree.father.father.father.namez;CreatedBy.user.family_tree.father.father.father.father.Name)r   )r   ri   )rV   r   Z
input_datarY   r   r   r   r   test_with_large_max_level  s"     
z,TestNestedToRecord.test_with_large_max_levelc          	   C   sF   t t2 ddlm} ddddddd	dg}|| W d Q R X d S )
Nr   )r   rG   rH   rI   )r@   rA   rJ   rK   rL   rM   )rT   Zassert_produces_warningFutureWarningZpandas.io.jsonr   )rV   r   rW   r   r   r   test_deprecated_import  s    z)TestNestedToRecord.test_deprecated_importc             C   s   ddddidddddiddd	d
didd}t |}dddg|_t|}tdddgddd	gdtjtjgtjdtjgtjtjdgd}t|| d S )NrG   ZFoor@   )r   r	   elementsrH   BarrA   rI   ZBazrJ   )r   rG   rH   g      ?g       @g      @)r   r	   z
elements.az
elements.bz
elements.c)r   r   r   r   r]   r   rT   rU   )rV   rn   srX   rY   r   r   r   test_series_non_zero_index  s    z-TestNestedToRecord.test_series_non_zero_index)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
  s:    "
%r   )r   numpyr]   r   Zpandasr   r   r   r   Zpandas._testingZ_testingrT   Zpandas.io.json._normalizer   Zfixturer   r&   r.   r8   rE   rF   r   r   r   r   r   <module>   s   '   