B
    `@                 @   s  d Z ddlmZ ddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZ dZdZdd	 Zejd
ddgdd Zdd Zejdddgddgfdddgddgfgdd Zdd Zdd Zdd Zejdd d!gd"d# Zejd$ddgejd
ddgddggd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zejd/ddge	d0d1gd2d3ggd4d5gd6fd5d7ge	d1d8gd3d9ggd5d7gd6fgd:d; Zejd
ddd<gd<ddggd=d> Z d?d@ Z!dAdB Z"dCdD Z#ejd
ddd<gd<ddggejdEe$dFe$dGgdHdI Z%dJdK Z&dLdM Z'ejd
dNdOgdPdQggdRdS Z(ejd
dTdUgdTdUggdVdW Z)dXdY Z*dZd[ Z+ejd/d\d] e	d^d_d`dadbddcdaddddaddfded] e	 fgdfdg Z,ejd
ddgdhd] gdidj Z-ejdkdldddge.ddme	dndodpgdndodpgdqdrdsggfdtdudvdwge. e	dd<ddddxgddyddddzgd<dxdyd<d<dcgd{fgd|d} Z/ejd~ddddge. e	ddxgddzgd<dcgdydbgddfddddge. de0dfdddge. de0dfddddge. de0dfde.ddudvdwdgde	ddxgddzgd<dcgdydbgddfdudvdwdge.ddudvdwdgdde0dfdudvdge.dudvdwdgdde0dfgdd Z1ejd
dudwgddggdd Z2dS )z]
Tests the usecols functionality during parsing
for all of the parsers defined in parsers.py
    )StringION)	Timestamp)	DataFrameIndexz['usecols' must either be list-like of all strings, all unicode, all integers or a callable.zAUsecols do not match columns, columns expected but not found: {0}c          	   C   sB   d}dddg}| }t jttd |jt||d W d Q R X d S )Nz<a,b,c
        1000,2000,3000
        4000,5000,6000
        r   b   )match)usecols)pytestraises
ValueError_msg_validate_usecols_argread_csvr   )all_parsersdatar	   parser r   R/home/dcms/DCMS/lib/python3.7/site-packages/pandas/tests/io/parser/test_usecols.py!test_raise_on_mixed_dtype_usecols   s
    
r   r	   )   r   )r   cc             C   sR   d}| }|j t||d}tddgddgddgd	d
ggddgd}t|| d S )Nz a,b,c
1,2,3
4,5,6
7,8,9
10,11,12)r	   r               	         r   r   )columns)r   r   r   tmassert_frame_equal)r   r	   r   r   resultexpectedr   r   r   test_usecols&   s
    (r#   c             C   s^   d}| }ddg}|j t||ddgdd}tddgd	d
gddgddgg|d}t|| d S )Nz a,b,c
1,2,3
4,5,6
7,8,9
10,11,12foobarr   r   r   )namesr	   headerr   r   r   r   r   r   r   )r   )r   r   r   r   r    )r   r   r   r&   r!   r"   r   r   r   test_usecols_with_names5   s    $r(   znames,usecolsr   r   r   r   ac             C   sV   d}| }|j t||d |d}tddgddgddgd	d
ggddgd}t|| d S )Nz1,2,3
4,5,6
7,8,9
10,11,12)r&   r'   r	   r   r   r   r   r   r   r   r   r   r   )r   )r   r   r   r   r    )r   r&   r	   r   r   r!   r"   r   r   r   test_usecols_relative_to_namesD   s
    (r*   c             C   s^   d}| }|j t|ddgd ddgd}tddgdd	gd
dgddggddgd}t|| d S )Nz1,2,3
4,5,6
7,8,9
10,11,12r)   r   r   r   )r&   r'   r	   r      r      r   
   r   )r   )r   r   r   r   r    )r   r   r   r!   r"   r   r   r   test_usecols_relative_to_names2T   s    (r.   c          	   C   sT   d}| }|j dkrdnd}tjt|d" |jt|ddgd dgd	 W d Q R X d S )
Nz1,2,3
4,5,6
7,8,9
10,11,12pythonzHNumber of passed names did not match number of header fields in the filez&Passed header names mismatches usecols)r   r)   r   r   )r&   r'   r	   )enginer
   r   r   r   r   )r   r   r   msgr   r   r   !test_usecols_name_length_conflictd   s    r2   c          	   C   s8   | }d}t jttd |jt|dd W d Q R X d S )Nz/foo, bar, baz
1000, 2000, 3000
4000, 5000, 6000)r   r$   )r	   )r
   r   r   r   r   r   )r   r   r   r   r   r   test_usecols_single_stringu   s    r3   r   za,b,c,d
1,2,3,4
5,6,7,8za,b,c,d
1,2,3,4,
5,6,7,8,c             C   sN   | }dddg}t ddgddgdd	gd
}|jt||dd}t|| d S )Nr)   r   dr   r   r   r,   r+   r   )r)   r   r4   F)r	   	index_col)r   r   r   r   r    )r   r   r   r	   r"   r!   r   r   r   test_usecols_index_col_false   s
    
r6   r5   c             C   sL   | }d}t dddgitddgddd}|jt|||d	}t|| d S )
Nza,b,c,d
A,a,1,one
B,b,2,twor   r   r   r)   r   )name)index)r	   r5   )r   r   r   r   r   r    )r   r	   r5   r   r   r"   r!   r   r   r   test_usecols_index_col_conflict   s
     r9   c             C   s\   | }d}t ddgddgdd}|ddg}|jt|ddd	gddgd
}t|| d S )Nza,b,c,d
A,a,1,one
B,b,2,twor)   r   r   r   )ZoneZtwo)r   r   r4   r   r4   )r	   r5   )r   Z	set_indexr   r   r   r    )r   r   r   r"   r!   r   r   r    test_usecols_index_col_conflict2   s    r:   c             C   sL   | }d}|j t|ddgd}tddgddgd	d
dgd}t|| d S )Nz%a,b,c
4,apple,bat,5.7
8,orange,cow,10r)   r   )r	   appleorangebatcow)r)   r   r+   r   )r8   )r   r   r   r   r    )r   r   r   r!   r"   r   r   r   test_usecols_implicit_index_col   s
    r?   c             C   sJ   | }d}|j t|ddd}tddgddgdd	d
gd}t|| d S )Nz-a  b  c
4  apple  bat  5.7
8  orange  cow  10z\s+)r)   r   )sepr	   r;   r<   r=   r>   r+   r   )r8   )r   r   r   r   r    )r   r   r   r!   r"   r   r   r   test_usecols_regex_sep   s
    rA   c             C   sJ   | }d}|j t|ddd}tddgddgdd	d
gd}t|| d S )Nz-a  b  c
4  apple  bat  5.7
8  orange  cow  10T)r)   r   )Zdelim_whitespacer	   r;   r<   r=   r>   r+   r   )r8   )r   r   r   r   r    )r   r   r   r!   r"   r   r   r   test_usecols_with_whitespace   s
    rB   zusecols,expectedi  i  i  i  20)r   r   1i  ip  c             C   s*   | }d}|j t||d}t|| d S )Nz#2,0,1
1000,2000,3000
4000,5000,6000)r	   )r   r   r   r    )r   r	   r"   r   r   r!   r   r   r   %test_usecols_with_integer_like_header   s    
rF   r   c             C   s`   d}| }ddgg}ddgt dt dgd}t|dd	gd
}|jt|||d}t|| d S )Nz1a,b,c,d,e
0,1,20140101,0900,4
0,1,20140102,1000,4r   r   r   z2014-01-01 09:00:00z2014-01-02 10:00:00)r)   c_drG   r)   )r   )r	   parse_dates)r   r   r   r   r   r    )r   r	   r   r   rH   colsr"   r!   r   r   r   test_usecols_with_parse_dates   s    
rJ   c       
      C   s   | }d}ddg}|d d  }dg}t tdtdtdgdd}dd	d
dgi}t||d}|jt||d|d |d}	t|	| d S )NzJ2008-02-07 09:40,1032.43
2008-02-07 09:50,1042.54
2008-02-07 10:00,1051.65datevaluesr   z2008-02-07 09:40z2008-02-07 09:50z2008-02-07 10:00)r7   gQ!@g\(J@gn@)r8   )rH   r5   r	   r'   r&   )r   r   r   r   r   r   r    )
r   r   r   r&   r	   rH   r8   rI   r"   r!   r   r   r   test_usecols_with_parse_dates2   s(    rM   c             C   sv   | }d}t d}dg}tddgdgdgdgdgd	gd
gdgdgd
}t||d}|jt|||d}t|| d S )Nz0a,b,c,d,e,f,g,h,i,j
2016/09/21,1,1,2,3,4,5,6,7,8
abcdefghijr   z
2016-09-21r   r   r   r+   r   r   r,   r   )
r)   r   r   r4   efghij)r   )r	   rH   )listr   r   r   r   r   r    )r   r   r   r	   rH   rI   r"   r!   r   r   r   test_usecols_with_parse_dates3
  s"    
rV   c          
   C   s|   d}t d}ddgg}| }ddgdgdgdgd	gd
gdgdgd	}t|dgt d d}|jt|||d}t|| d S )Nz0a,b,c,d,e,f,g,h,i,j
2016/09/21,1,1,2,3,4,5,6,7,8rN   r   r   z2016/09/21 1r   r   r+   r   r   r,   r   )	a_br   r4   rO   rP   rQ   rR   rS   rT   rW   Zcdefghij)r   )r	   rH   )rU   r   r   r   r   r    )r   r   r	   rH   r   rI   r"   r!   r   r   r   test_usecols_with_parse_dates4%  s     

rX   r&   ZabcdeZacdc       	      C   sb   d}ddgg}| }ddgt dt dgd}t|dd	gd
}|jt||||d}t|| d S )Nz'0,1,20140101,0900,4
0,1,20140102,1000,4r   r   r   z2014-01-01 09:00:00z2014-01-02 10:00:00)r)   rG   rG   r)   )r   )r&   rH   r	   )r   r   r   r   r   r    )	r   r	   r&   srH   r   rI   r"   r!   r   r   r   'test_usecols_with_parse_dates_and_names<  s    
rZ   c             C   sP   d}| }ddddddddd	}t |}|jt|d
dgd}t|| d S )NzPAAA,BBB,CCC,DDD
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,ag?g$@gr.-@)r   r   r   r   r   r,   )AAABBBr[   r\   )r	   )r   r   r   r   r    )r   r   r   exp_datar"   r!   r   r   r   !test_usecols_with_unicode_stringsW  s    
r^   c             C   sP   d}| }ddddddddd	}t |}|jt|d
dgd}t|| d S )NzHA,B,C,D
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,ag?g$@gr.-@)r   r   r   r   r   r,   )ABr_   r`   )r	   )r   r   r   r   r    )r   r   r   r]   r"   r!   r   r   r   -test_usecols_with_single_byte_unicode_stringsi  s    
ra   r[   s   BBBs   AAAr\   c          	   C   s8   d}| }t jttd |jt||d W d Q R X d S )NzPAAA,BBB,CCC,DDD
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,a)r   )r	   )r
   r   r   r   r   r   )r   r	   r   r   r   r   r   (test_usecols_with_mixed_encoding_strings{  s    rb   u	   あああu   いいc             C   sL   d}| }ddddddddd	}t |}|jt||d
}t|| d S )Nuh   あああ,いい,ううう,ええええ
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,ag?g$@gr.-@)r   r   r   r   r   r,   )u	   あああu   いい)r	   )r   r   r   r   r    )r   r	   r   r   r]   r"   r!   r   r   r   'test_usecols_with_multi_byte_characters  s    
rc   c             C   s2   d}t  }| }|jt|t d}t|| d S )Nza,b,c
1,2,3
4,5,6)r	   )r   r   r   setr   r    )r   r   r"   r   r!   r   r   r   test_empty_usecols  s
    re   c             C   sJ   | }d}t ddg}tddgg|d}|jt||d}t|| d S )Nza,b,c
1,2,3r)   r   r   r   )r   )r	   )nparrayr   r   r   r   r    )r   r   r   r	   r"   r!   r   r   r   test_np_array_usecols  s    rh   c             C   s   |   dkS )N)r[   r\   ZDDD)upper)xr   r   r   <lambda>      rk   g?g$@gr.-@)r   r   r   r   r,   )ZAaAZbBbZdddc             C   s   dS )NFr   )rj   r   r   r   rk     rl   c             C   s*   d}| }|j t||d}t|| d S )NzPAaA,bBb,CCC,ddd
0.056674973,8,True,a
2.613230982,2,False,b
3.568935038,7,False,a)r	   )r   r   r   r    )r   r	   r"   r   r   r!   r   r   r   test_callable_usecols  s    rm   c             C   s   | dkS )N)r)   r   r   )rj   r   r   r   rk     rl   c             C   sN   d}| }dddg}t ddgtjdgd}|jt|||d}t|| d S )	Nz	1,2
1,2,3r)   r   r   r   r   )r)   r   )r&   r	   )r   rf   nanr   r   r   r    )r   r	   r   r   r&   r"   r!   r   r   r   test_incomplete_first_row  s    
ro   zdata,usecols,kwargs,expectedz19,29,39
19,29,39
10,20,30,40)r'         '   r-         z4A,B,C
1,2,3
3,4,5
1,2,4,5,1,6
1,2,3,,,1,
1,2,3
5,6,7r_   r`   Cr   r+   r   )r_   r`   ru   c             C   s.   | }|j t|fd|i|}t|| d S )Nr	   )r   r   r   r    )r   r   r	   kwargsr"   r   r!   r   r   r   test_uneven_length_cols  s    rw   zusecols,kwargs,expected,msgr4   )r)   r   r   r4   rP   z\['f'\]rQ   z\[('f', 'g'|'g', 'f')\]D)r'   r&   )r_   r`   ru   rx   )r&   c          	   C   sl   d}|j |d | }|d krJtjt|d |jt|f| W d Q R X n|jt|f|}t|| d S )Nza,b,c,d
1,2,3,4
5,6,7,8)r	   )r   )updater
   r   r   r   r   r   r    )r   r	   rv   r"   r1   r   r   r!   r   r   r   %test_raises_on_usecols_names_mismatch  s    ,rz   c       	      C   sv   | j dkr&d}|jtjj|td d}ddddg}| }|jt|d	||d
}t	ddgddgd}t
|| d S )Nr   z=see gh-16469: works on the C engine but not the Python engine)reasonr   za,b,c,d
1,2,3,4
5,6,7,8r_   r`   ru   rx   r   )r'   r&   r	   r   r   r   r,   )r_   ru   )r0   nodeZ
add_markerr
   markZxfailr   r   r   r   r   r    )	r   r	   requestr{   r   r&   r   r!   r"   r   r   r   /test_usecols_subset_names_mismatch_orig_columns1  s    
r   )3__doc__ior   numpyrf   r
   Zpandas._libs.tslibr   Zpandasr   r   Zpandas._testingZ_testingr   r   Z_msg_validate_usecols_namesr   r}   Zparametrizer#   r(   r*   r.   r2   r3   r6   r9   r:   r?   rA   rB   rF   rJ   rM   rV   rX   rU   rZ   r^   ra   rb   rc   re   rh   rm   ro   dictrw   formatrz   r   r   r   r   r   <module>   s   .$

	"0&!""	
" $
 



 

