B
    `$                 @   s   d Z ddlmZmZ ddlZddlZddlZddlm	  m
Z ddlmZ ddlmZ ddlmZ ddlmZ ejjejddd	d
dgejdddgejdddgdd Zejdd Zejdd Zejde G dd dZdS )zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
    )BytesIOStringION)	DataFrame)read_feather)read_csvzcompress_type, extension)gzipz.gz)bz2z.bz2)zipz.zip)xzz.xzmodeexplicitinferenginepythoncc             C   s   t | |||| d S )N)check_compressed_urls)salaries_tablecompress_type	extensionr   r    r   R/home/dcms/DCMS/lib/python3.7/site-packages/pandas/tests/io/parser/test_network.pytest_compressed_urls   s    r   c             C   s8   d}|| }|dkr|}t |d||d}t||  d S )NzXhttps://github.com/pandas-dev/pandas/raw/master/pandas/tests/io/parser/data/salaries.csvr   	)sepcompressionr   )r   tmassert_frame_equal)r   r   r   r   r   base_urlurlZ	url_tabler   r   r   r      s    r   c             C   s   t | ddddS )z DataFrame with the tips dataset.iodatacsvztips.csv)r   )Zdatapathr   r   r   tips_df1   s    r"   s3_resourcec               @   s   e Zd Zeddd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd Zdd Zeddd Zdd Zdd Zdd  Zedd!d" Zd#S )$TestS3s3fsc             C   sv   xDdD ]<\}}t d| |d}t|ts,t|jr6tt|| qW t d}t|ts\t|jrftt|| d S )N)) N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csv)r   zs3://cant_get_it/tips.csv)r   
isinstancer   AssertionErroremptyr   r   )selfr"   extcompdfr   r   r   test_parse_public_s3_bucket:   s    

z"TestS3.test_parse_public_s3_bucketc             C   s>   t ddd}t|tst|jr$tt|jd d | d S )Nzs3n://pandas-test/tips.csv
   )nrows)r   r'   r   r(   r)   r   r   iloc)r*   r"   r-   r   r   r   test_parse_public_s3n_bucketK   s    
z#TestS3.test_parse_public_s3n_bucketc             C   s>   t ddd}t|tst|jr$tt|jd d | d S )Nzs3a://pandas-test/tips.csvr/   )r0   )r   r'   r   r(   r)   r   r   r1   )r*   r"   r-   r   r   r   test_parse_public_s3a_bucketS   s    
z#TestS3.test_parse_public_s3a_bucketc             C   sV   xPdD ]H\}}t d| d|d}t|ts.t|jr8tt|jd d | qW d S )N))r&   N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csvr/   )r0   r   )r   r'   r   r(   r)   r   r   r1   )r*   r"   r+   r,   r-   r   r   r   !test_parse_public_s3_bucket_nrowsZ   s
    
z(TestS3.test_parse_public_s3_bucket_nrowsc       	      C   s   d}xdD ]|\}}t d| ||d}|j|ks2txRdD ]J}| }t|tsRt|jr\t|j|| ||d   }t	|| q8W q
W d S )N   ))r&   N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csv)	chunksizer   )r         r7   )
r   r6   r(   	get_chunkr'   r   r)   r1   r   r   )	r*   r"   r6   r+   r,   	df_readeri_chunkr-   true_dfr   r   r   #test_parse_public_s3_bucket_chunkeda   s    

z*TestS3.test_parse_public_s3_bucket_chunkedc       	      C   s   d}xdD ]~\}}t d| ||dd}|j|ks4txRdD ]J}| }t|tsTt|jr^t|j|| ||d   }t	|| q:W q
W d S )Nr5   ))r&   N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csvr   )r6   r   r   )r   r7   r8   r7   )
r   r6   r(   r9   r'   r   r)   r1   r   r   )	r*   r"   r6   r+   r,   r:   r;   r-   r<   r   r   r   *test_parse_public_s3_bucket_chunked_pythonr   s    

z1TestS3.test_parse_public_s3_bucket_chunked_pythonc             C   sL   xFdD ]>\}}t d| d|d}t|ts.t|jr8tt|| qW d S )N))r&   N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csvr   )r   r   )r   r'   r   r(   r)   r   r   )r*   r"   r+   r,   r-   r   r   r   "test_parse_public_s3_bucket_python   s    
z)TestS3.test_parse_public_s3_bucket_pythonc             C   sH   xBdD ]:}t d| ddd}t|ts*t|jr4tt|| qW d S )N)r&   z.gzz.bz2zs3://pandas-test/tips.csvr   r   )r   r   )r   r'   r   r(   r)   r   r   )r*   r"   r+   r-   r   r   r   test_infer_s3_compression   s    

z TestS3.test_infer_s3_compressionc             C   sX   xRdD ]J\}}t d| dd|d}t|ts0t|jr:tt|jd d | qW d S )N))r&   N)z.gzr   )z.bz2r   zs3://pandas-test/tips.csvr   r/   )r   r0   r   )r   r'   r   r(   r)   r   r   r1   )r*   r"   r+   r,   r-   r   r   r   (test_parse_public_s3_bucket_nrows_python   s    
z/TestS3.test_parse_public_s3_bucket_nrows_pythonc          	   C   s@   t t td W d Q R X t t td W d Q R X d S )Nzs3://nyqpug/asdf.csvzs3://cant_get_it/file.csv)pytestraisesIOErrorr   )r*   r   r   r   test_read_s3_fails   s    zTestS3.test_read_s3_failsc          	   C   s<   dd l }t|jjf}tj|dd |d W d Q R X d S )Nr   z#The specified bucket does not exist)matchz/s3://an_s3_bucket_data_doesnt_exit/not_real.csv)botocoreFileNotFoundError
exceptionsClientErrorrB   rC   to_csv)r*   r"   rG   errorr   r   r   test_write_s3_csv_fails   s    zTestS3.test_write_s3_csv_failsZpyarrowc          	   C   s<   dd l }t|jjf}tj|dd |d W d Q R X d S )Nr   z#The specified bucket does not exist)rF   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquet)rG   rH   rI   rJ   rB   rC   Z
to_parquet)r*   r"   rG   rL   r   r   r   test_write_s3_parquet_fails   s    z"TestS3.test_write_s3_parquet_failsc             C   sZ   |j jjddd}tt|d  dd}t|ts8t|j	rBtt|}t
|| d S )Nzpandas-testztips.csv)BucketKeyBodyutf8)encoding)metaclientZ
get_objectr   r   readr'   r   r(   r)   r   r   )r*   r#   Z	tips_fileZ	s3_objectresultexpectedr   r   r   $test_read_csv_handles_boto_s3_object   s    

z+TestS3.test_read_csv_handles_boto_s3_objectc          	   C   s   dd l }ttjddtdd}t }|| t|	 
d}|djd|d	 |j  |jtjd
d* tddd ddd |jD kstW d Q R X d S )Nr   i    Zabcd)columnszutf-8zpandas-testzlarge-file.csv)rP   rQ   r%   )loggerzs3://pandas-test/large-file.csvr5   )r0   )r   i  T c             s   s   | ]}|j d d V  qdS )N)args).0xr   r   r   	<genexpr>   s    z8TestS3.test_read_csv_chunked_download.<locals>.<genexpr>)r%   r   nprandomZrandnlistr   rK   r   getvalueencoderO   Z
put_objectZS3FileSystemZclear_instance_cacheZat_levelloggingDEBUGr   recordsr(   )r*   r#   Zcaplogr%   r-   Zstr_bufbufr   r   r   test_read_csv_chunked_download   s    

z%TestS3.test_read_csv_chunked_downloadc             C   s   t d}t|| d S )Nzs3://pandas-test/tips#1.csv)r   r   r   )r*   r"   rW   r   r   r   test_read_s3_with_hash_in_key   s    z$TestS3.test_read_s3_with_hash_in_keyc             C   s    t |}t d}t|| d S )Nz's3://pandas-test/simple_dataset.feather)r   r   r   )r*   Zfeather_filerX   resr   r   r   test_read_feather_s3_file_path   s    z%TestS3.test_read_feather_s3_file_pathN)__name__
__module____qualname__tdZ
skip_if_nor.   r2   r3   r4   r=   r>   r?   r@   rA   rE   rM   rN   rY   rk   rl   rn   r   r   r   r   r$   7   s    			r$   )__doc__r   r   r   rg   numpyrb   rB   Zpandas.util._test_decoratorsutilZ_test_decoratorsrr   Zpandasr   Zpandas._testingZ_testingr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   marknetworkZparametrizer   r   Zfixturer"   ZusefixturesZskip_if_not_us_localer$   r   r   r   r   <module>   s&    
