B
    `b                 @   s  d Z ddlmZ ddlmZ ddlZddlZddlmZ ddl	m
Z
mZ ddlmZ ejdeeddgd	d
 Zdd Zdd Zejddedgde
dddgdddggdddgdfdeddgde
ddd ggd!d"d#gdfd$edddgde
d%d&ggd'd(gdfgd)d* Zd+d, Zejd-d.dd/dgdd0dggfd1dd2dgdd3dggfd4dd5dgdd6dggfgd7d8 Zejd9d:d;d<gd=d> Zd?d@ ZejdAe e
dBddCgifeddDgdEe
dDddCgifgdFdG ZdHdI ZdJdK ZdS )Lzj
Tests that skipped rows are properly handled during
parsing for all of the parsers defined in parsers.py
    )datetime)StringION)EmptyDataError)	DataFrameIndexskiprows   c             C   s|   | }d}|j t||d ddd}ttdddtdddtdddgdd	}ttd
dddddg|d}t	|| d S )Nzr#foo,a,b,c
#foo,a,b,c
#foo,a,b,c
#foo,a,b,c
#foo,a,b,c
#foo,a,b,c
1/1/2000,1.,2.,3.
1/2/2000,4,5,6
1/3/2000,7,8,9
r   T)r   header	index_colparse_datesi           )nameg      ?g      $@)r   r   )columnsindex)
read_csvr   r   r   r   nparangereshapetmassert_frame_equal)all_parsersr   parsertextresultr   expected r   S/home/dcms/DCMS/lib/python3.7/site-packages/pandas/tests/io/parser/test_skiprows.pytest_skip_rows_bug   s    
( r   c             C   sl   | }dd dd tdD  }dd dd dD  }|jt|dd	gd
}|t|}t|| d S )Nza,b,c

c          	   S   s0   g | ](}d  t|t|d t|d gqS ),r   r   )joinstr).0ir   r   r   
<listcomp>1   s    z'test_deep_skip_rows.<locals>.<listcomp>
   c          	   S   s0   g | ](}d  t|t|d t|d gqS )r!   r   r   )r"   r#   )r$   r%   r   r   r   r&   4   s    )r   r   r   r      r      	   r   r)   )r   )r"   ranger   r   r   r   )r   r   dataZcondensed_datar   Zcondensed_resultr   r   r   test_deep_skip_rows-   s    r-   c             C   s|   | }d}|j t|dd ddd}ttdddtdddtddd	gdd
}ttdddddd	g|d}t	|| d S )Nz^#foo,a,b,c
#foo,a,b,c

#foo,a,b,c
#foo,a,b,c

1/1/2000,1.,2.,3.
1/2/2000,4,5,6
1/3/2000,7,8,9
r   r   T)r   r	   r
   r   i  r   r   r   )r   g      ?g      $@)r   r   )r   r   )
r   r   r   r   r   r   r   r   r   r   )r   r   r   r,   r   r   r   r   r   test_skip_rows_blank<   s    
( r.   zdata,kwargs,expectedzKid,text,num_lines
1,"line 11
line 12",2
2,"line 21
line 22",2
3,"line 31",1r   )r   r   zline 21
line 22r   zline 31idr   	num_lines)r   z+a,b,c
~a
 b~,~e
 d~,~f
 f~
1,2,~12
 13
 14~~)	quotecharr   za
 bze
 dzf
 fabczdText,url
~example
 sentence
 one~,url1
~example
 sentence
 two~,url2
~example
 sentence
 three~,url3zexample
 sentence
 twourl2Texturlc             C   s&   | }|j t|f|}t|| d S )N)r   r   r   r   )r   r,   kwargsr   r   r   r   r   r   test_skip_row_with_newlineV   s    #r:   c             C   sR   | }d}dddgdddgg}t |ddd	gd
}|jt|dgd}t|| d S )NzYid,text,num_lines
1,"line '11' line 12",2
2,"line '21' line 22",2
3,"line '31' line 32",1r   zline '21' line 22r   zline '31' line 32r   r/   r   r0   )r   )r   )r   r   r   r   r   )r   r   r,   exp_datar   r   r   r   r   test_skip_row_with_quote~   s    r<   zdata,exp_dataz\id,text,num_lines
1,"line 
'11' line 12",2
2,"line 
'21' line 22",2
3,"line 
'31' line 32",1zline 
'21' line 22zline 
'31' line 32z\id,text,num_lines
1,"line '11
' line 12",2
2,"line '21
' line 22",2
3,"line '31
' line 32",1zline '21
' line 22zline '31
' line 32zbid,text,num_lines
1,"line '11
' 	line 12",2
2,"line '21
' 	line 22",2
3,"line '31
' 	line 32",1zline '21
' 	line 22zline '31
' 	line 32c             C   s:   | }|j t|dgd}t|dddgd}t|| d S )Nr   )r   r/   r   r0   )r   )r   r   r   r   r   )r   r,   r;   r   r   r   r   r   r   $test_skip_row_with_newline_and_quote   s    r=   line_terminatorr    z
c          	   C   s   | }d ddddg}tdddd	d
gdddd
dgddddd
ggdddddgd}|jdkrl|dkrltd |d|}|jt|dddddddgd}t	|| d S )Nr    zSMOSMANIA ThetaProbe-ML2X z2007/01/01 01:00   0.2140 U M z2007/01/01 02:00   0.2141 M O z2007/01/01 04:00   0.2142 D M z
2007/01/01z01:00g1Zd?UMz02:00gk	g?Oz04:00gBfj?DdatetimevarflagZoflag)r   pythonr?   z+'CR' not respect with the Python parser yetr   T)r   Zdelim_whitespacenames)
r"   r   Zenginepytestskipreplacer   r   r   r   )r   r>   r   r,   r   r   r   r   r   test_skiprows_lineterminator   s(    
rM   c             C   s8   | }d}t ddgi}|jt|dd}t|| d S )Nz	a"
b"
a
1r3   r   r   )r   )r   r   r   r   r   )r   r   r,   r   r   r   r   r   test_skiprows_infield_quote   s
    rN   zkwargs,expected1   Zfoo)r	   rI   c             C   s6   | }d}|j t|fddd i|}t|| d S )Nza
1
2
3
4
5r   c             S   s   | d dkS )Nr   r   r   )xr   r   r   <lambda>       z)test_skip_rows_callable.<locals>.<lambda>)r   r   r   r   )r   r9   r   r   r,   r   r   r   r   test_skip_rows_callable   s    rT   c          	   C   s@   | }d}d}t jt|d |jt|dd d W d Q R X d S )Nza
1
2
3
4
5zNo columns to parse from file)matchc             S   s   dS )NTr   )rQ   r   r   r   rR      rS   z)test_skip_rows_skip_all.<locals>.<lambda>)r   )rJ   raisesr   r   r   )r   r   r,   msgr   r   r   test_skip_rows_skip_all   s
    rX   c          	   C   s@   d}| }d}t jt|d |jt|dd d W d Q R X d S )Nzby zeroza
1
2
3
4
5)rU   c             S   s   dd S )Nr   r   r   )rQ   r   r   r   rR      rS   z-test_skip_rows_bad_callable.<locals>.<lambda>)r   )rJ   rV   ZeroDivisionErrorr   r   )r   rW   r   r,   r   r   r   test_skip_rows_bad_callable   s
    rZ   )__doc__r   ior   numpyr   rJ   Zpandas.errorsr   Zpandasr   r   Zpandas._testingZ_testingr   markZparametrizelistr+   r   r-   r.   dictr:   r<   r=   rM   rN   rT   rX   rZ   r   r   r   r   <module>   sR   "
$"#
*	