a
    PfQ                     @  s  d Z ddlmZ ddlmZ ddlZddlZddlZddlmZ ddlZddl	Z	ddl
Z
ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddlZddlmZmZ dd	lmZ dd
lmZ ddlm  m Z! ddl"Z#ddl"m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m	  m.Z/ ddl0m1Z1m2Z2 dd Z3ej45ddde6de7de8dedg de7d9dde7d9ddddd e:dj;ddde:dj;ddde:dj;ddddddddf gdd Z<dd  Z=d!d" Z>d#d$ Z?d%d&d'd(d)Z@ej45d*ejAe>d+d,ejAe#jBd-d,ejAeje#jBejCd.d/d,ejAeje#jBd0d.d1d,ejAeje#jBd2d.d3d,gej45d4e#jBe>gej45d5e@e, d6d7 ZDd8d9 ZEd:d; ZFejGd<d= ZHG d>d? d?ZIG d@dA dAZJej45dBdCgdDe%g dEfdFe#jKLg dEg dGgfgdHdI ZMdJdK ZNej45dLg dMdNdO ZOdPdQ ZPG dRdS dSejQZRdTdU ZSdVdW ZTdXdY ZUdZd[ ZVej45d\d]d^d\ed_d`d`dagdbdc ZWddde ZXdfdg ZYe!jZdhdi Z[ej45djej\ejCgdkdl Z]dmdn Z^dodp Z_dS )qa<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )annotations)arrayN)partial)Path)Any)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)	DataFrameIndexSeriesperiod_range)create_pickle_data)DayMonthEndc                 C  s   t |trt||  d S |dr2t| | nP|dkrb|tju rT| tju s`J q| |ksJ n ttd| dtj	}|| | d S )NZsp_	timestampZassert__equal)

isinstancer   tmassert_index_equal
startswithZassert_equalpdZNaTgetattrZassert_almost_equal)resultexpectedtypZ
comparator r   T/var/www/ai-form-bot/venv/lib/python3.9/site-packages/pandas/tests/io/test_pickle.pycompare_element>   s    


r   datas   123s   123456I         B)r%   r$   r$      )r%      C)orderFc                 C  s   t | }t| d}||ks"J t| ttfr>|| u sJ nBt|tr|jdksVJ |jdksdJ |jsnJ |j	|j
fksJ d S )NAr#   r&   )r
   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r    r   r   r   r   r   test_flatten_bufferT   s    

r6   c           
   	   C  s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkrR|dkrRt
|| |jj|jjksJ |jjjrJ t
|dk|dk |jj}|td tdksJ |tjdd }	t|	tjsJ |	tjddd	ksJ |tjdd
 }	t|	tjs8J |	tjdddksJ qR|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d qR|dkr|dv rt
|| qR|dkr|dv rt
|| qRt||| qRq@q d S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*Zseriestsr   r#   r$   )hours)daysr8   )nanoseconds)r9   r:   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   __file__parentglobr   read_pickleitemsr   assert_series_equalr;   freq	normalizer   Z	Timedeltar   r   r   freqstrshiftassert_frame_equalr   )
datapathlegacy_pickler    r   Zdvdtr   r   rG   resr   r   r   test_picklesq   s@    

rP   c                 C  s>   t |d }tj| |dd W d    n1 s00    Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler   s    r^   c                 C  sB   t | d$}|d t|W  d    S 1 s40    Y  d S )Nrbr   )rW   seekrX   load)r\   r]   r   r   r   python_unpickler   s    
rb   dictzlist[tuple[str, Any]])r    returnc                 C  s   dd |   D S )zFlatten create_pickle_datac                 S  s&   g | ]\}}|  D ]}||fqqS r   )values).0r   Zexamplesexampler   r   r   
<listcomp>   s   zflatten.<locals>.<listcomp>)rE   )r    r   r   r   flatten   s    ri   pickle_writerpython)idZpandas_proto_defaultrT   Zpandas_proto_highestr(   Zpandas_proto_4   Zpandas_proto_5writerztyp, expectedc              	   C  s   t  }||| t|}t|||  t|}t|||  t|dd$}||| |d W d    n1 st0    Y  t|dd$}t|}|d W d    n1 s0    Y  t|||  W d    n1 s0    Y  d S )NrR   moder   r_   )r   ensure_cleanr   rD   r   rb   rW   r`   )r   r   rj   rn   r\   r   handler   r   r   test_round_trip_current   s    



(
(rs   c                  C  s`   t dtdd ttdtdtdd tdD tdd	} t	| j
tj}t| | d S )
N皙?x      r(   ABCDZdtypec                 S  s   g | ]}d | qS zi-r   rf   ir   r   r   rh          z,test_pickle_path_pathlib.<locals>.<listcomp>rw   columnsr;   )r   nparangereshaper   listobjectranger   round_trip_pathlib	to_pickler   rD   rK   dfr   r   r   r   test_pickle_path_pathlib   s    r   c                  C  s`   t dtdd ttdtdtdd tdD tdd	} t	| j
tj}t| | d S )
Nrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh      r}   z.test_pickle_path_localpath.<locals>.<listcomp>rw   r~   )r   r   r   r   r   r   r   r   r   Zround_trip_localpathr   r   rD   rK   r   r   r   r   test_pickle_path_localpath   s    r   c                   C  s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s    r   c                   @  sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C  s  |d u rt || d S |dkr0t|d}n4|dkrHt|d}n|dkrtj|dtjd$}|	|t
j| W d    n1 s0    Y  n|dkrt|d^}tj|dd0}||t
j|}||| W d    n1 s0    Y  W d    n1 s0    Y  nF|d	kr6t |d}n.|d
krRtd|d}nd| }t||dvrt|dD}| |	|  W d    n1 s0    Y  W d    n1 s0    Y  d S )Ngzipwbz2zipcompressiontarr_   ro   xzZzstdZ	zstandardrR   zUnrecognized compression type: )r   r   )shutilcopyfiler   rW   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosr\   basenametarfile
gettarinfoaddfiler   r	   
ValueErrorread)	selfZsrc_path	dest_pathr   fr]   r   tarinfomsgr   r   r   compress_file   s2    4
L



zTestCompression.compress_filec                 C  s@  |}|d }|d }t |
}t |}tdtdd ttdtdtdd	 t	d
D tdd}|j
||d t j||dH}	t|d}
|
|	  W d    n1 s0    Y  W d    n1 s0    Y  tj|d d}t || W d    n1 s0    Y  W d    n1 s20    Y  d S )N.compressed.rawrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh   '  r}   z7TestCompression.test_write_explicit.<locals>.<listcomp>rw   r~   r   rR   )r   rq   r   r   r   r   r   r   r   r   r   decompress_filerW   r   r   r   rD   rK   )r   r   r   basepath1path2p1p2r   r   r]   df2r   r   r   test_write_explicit  s    Jz#TestCompression.test_write_explicitr   ) NonebadZ7zc              	   C  s   t jtdd t|^}tdtdd t	t
dtdt	dd	 td
D tdd}|j||d W d    n1 s~0    Y  W d    n1 s0    Y  d S )NzUnrecognized compression typematchrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh   >  r}   z;TestCompression.test_write_explicit_bad.<locals>.<listcomp>rw   r~   r   )r?   Zraisesr   r   rq   r   r   r   r   r   r   r   r   r   )r   r   r   r\   r   r   r   r   test_write_explicit_bad7  s    z'TestCompression.test_write_explicit_badc                 C  sL  |}|| }|d }| j | }t|}t|}tdtdd t	t
dtdt	dd td	D tdd
}	|	| tj||dH}
t|d}||
  W d    n1 s0    Y  W d    n1 s0    Y  tj|d d}t|	| W d    n1 s0    Y  W d    n1 s>0    Y  d S )Nr   rt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh   L  r}   z4TestCompression.test_write_infer.<locals>.<listcomp>rw   r~   r   rR   )_extension_to_compressiongetlowerr   rq   r   r   r   r   r   r   r   r   r   r   rW   r   r   r   rD   rK   )r   compression_extr   r   r   r   r   r   r   r   r   r]   r   r   r   r   test_write_inferB  s     
Jz TestCompression.test_write_inferc           
   	   C  s   |}|d }|d }t |}t |}tdtdd ttdtdtdd	 t	d
D tdd}|j
|d d | j|||d tj||d}	t ||	 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   r   rt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh   e  r}   z6TestCompression.test_read_explicit.<locals>.<listcomp>rw   r~   r   )r   rq   r   r   r   r   r   r   r   r   r   r   r   rD   rK   )
r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_explicit\  s    z"TestCompression.test_read_explicitc              	   C  s   |}|d }|| }| j | }t|}t|}tdtdd t	t
dtdt	dd td	D tdd
}	|	j|d d | j|||d t|}
t|	|
 W d    n1 s0    Y  W d    n1 s0    Y  d S )Nr   rt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh   |  r}   z3TestCompression.test_read_infer.<locals>.<listcomp>rw   r~   r   )r   r   r   r   rq   r   r   r   r   r   r   r   r   r   r   r   rD   rK   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_inferr  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   r?   markparametrizer   r   r   r   r   r   r   r   r      s   

r   c                   @  s&   e Zd Zejdg ddd ZdS )TestProtocolrU   )rS   r   r#   r$   c                 C  s   t |t}tdtdd ttdtdtdd t	dD tdd	}|j
||d
 t|}t || W d    n1 s0    Y  d S )Nrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh     r}   z*TestProtocol.test_read.<locals>.<listcomp>rw   r~   rT   )r   rq   r   r   r   r   r   r   r   r   r   r   rD   rK   )r   rU   r   r\   r   r   r   r   r   	test_read  s    
zTestProtocol.test_readN)r   r   r   r?   r   r   r   r   r   r   r   r     s   r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)r,   r&   r)   c                 C  s*   | ddd|}t |}t|j| d S )Nior    rX   )r   rD   r   r   r   )rL   r   r   r\   r   r   r   r   test_unicode_decode_error  s    
r   c               	   C  s   t  } tdtdd ttdtdtdd t	dD tdd	}t
| d
}|| W d    n1 st0    Y  t
| d}t|}W d    n1 s0    Y  t || W d    n1 s0    Y  d S )Nrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh     r}   z0test_pickle_buffer_roundtrip.<locals>.<listcomp>rw   r~   rR   r_   )r   rq   r   r   r   r   r   r   r   r   rW   r   r   rD   rK   )r\   r   r]   r   r   r   r   test_pickle_buffer_roundtrip  s    
((r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                   s   dd }G dd d t   fdd}tdtdd	 ttd
tdtdd t	dD tdd}|| | 
d| t|}t || W d    n1 s0    Y  d S )Nc                 S  s>   t |d }tj| |dd W d    n1 s00    Y  d S rQ   rV   rZ   r   r   r   r^     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @  s:   e Zd ZddddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponser   rd   c                 S  s.   t |d| _d|v r ddi| _n
ddi| _d S )Nr_   r   zContent-Encodingr   )rW   fileheaders)r   r\   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S  s   | S Nr   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W  s   |    d S r   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S  s
   | j  S r   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S  s
   | j  S r   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                    s    S r   r   )r   kwargsr   r\   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readrt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh     r}   z/test_pickle_generalurl_read.<locals>.<listcomp>rw   r~   zurllib.request.urlopen)r   rq   r   r   r   r   r   r   r   r   setattrr   rD   rK   )Zmonkeypatchr   r^   r   r   r   r   r   r   test_pickle_generalurl_read  s    


r   c                  C  s   t d t t d} tdtdd tt	dt
dtdd	 td
D t
dd}||  t| }t|| W d    n1 s0    Y  d S )NZfsspeczmemory://mockfilert   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh     r}   z0test_pickle_fsspec_roundtrip.<locals>.<listcomp>rw   r~   )r?   Zimportorskipr   rq   r   r   r   r   r   r   r   r   r   r   rD   rK   )r   r   r   r   r   r   test_pickle_fsspec_roundtrip  s    



r   c                   @  s   e Zd ZddddZdS )MyTzr   r   c                 C  s   d S r   r   r   r   r   r   r     s    zMyTz.__init__N)r   r   r   r   r   r   r   r   r     s   r   c                  C  sD   t tdt f} t| }t|d | d  t|d ts@J d S )Nry   r   r#   )r   r   r   r   round_trip_picklerF   r   )r   r   r   r   r   test_read_pickle_with_subclass  s    
r   c                 C  s   t dtdd ttdtdtdd tdD tdd	}t	 *}|j
|| d
 t| }W d   n1 sx0    Y  t }|j
|| d
 |d | |ks| dv sJ tj|| d
}|d t|| dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    rt   ru   rv   rx   ry   c                 S  s   g | ]}d | qS rz   r   r{   r   r   r   rh     r}   z9test_pickle_binary_object_compression.<locals>.<listcomp>rw   r~   r   Nr   )r   r   r   )r   r   r   r   r   r   r   r   r   rq   r   r   
read_bytesr   BytesIOr`   getvaluer   rD   rK   )r   r   r\   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    
*

r   c                 C  s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S  s   t | }t | | d S r   )r   r   rK   )r>   	unpickledr   r   r   _test_roundtrip7  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr>   r   r   r   r   +test_pickle_dataframe_with_multilevel_index0  s    
r   c                  C  sH   t dddd} ttjdt| | }t|}|j	j
dksDJ d S )Nz1/1/2011z1/1/2012r<   )rG   r$   )r   r   r   randomZdefault_rngZstandard_normallenr   r   r;   rI   )Zprngr7   Znew_tsr   r   r   "test_pickle_timeseries_periodindexA  s    
r   namei	  g     H@i     )r#   r$   c                 C  s0   t ttjdtjd| d}|j| ks,J d S )N
   ry   )r   )r   r   r   r   r   Zfloat64r   )r   r   r   r   r   test_pickle_preserve_nameI  s    r   c                 C  s   t | }t ||  d S r   r   r   rF   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimesQ  s    
r   c                 C  s   t | }t ||  d S r   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_stringsV  s    
r   c                  C  sf   t tddjdg } t| }|jjd jdks:J |jjd j	dksPJ t
|dg |  d S )Nabccategoryr   r#   )r#   T)r   r   ZastypeZilocr   r   Z_mgrblocksr1   r4   rF   )ZserrO   r   r   r    test_pickle_preserves_block_ndim[  s
    
r   rU   c                 C  s>   t td}tt|j| |dttj|d}t|| d S )Ni )rU   r   r   )	r   r   r   r   r   r   r   rD   rK   )rU   r   r   r   r   r   r   %test_pickle_big_dataframe_compressionh  s    r  c                 C  sf   | t tjdddd}t|d}t|}W d    n1 s@0    Y  tg g d}t|| d S )Nr    rM   z1.2.4zempty_frame_v1_2_4-GH#42345.pklr_   )r;   r   )	r   rA   rB   rW   rX   ra   r   r   rK   )rL   r\   fdr   r   r   r   r   #test_pickle_frame_v124_unpickle_130s  s    (r  c                  C  sZ   t dg di} d}tjt|d$ t }| |d W d    n1 sL0    Y  d S )Nr   r"   zpStarting with pandas version 3.0 all arguments of to_pickle except for the argument 'path' will be keyword-only.r   Zinfer)r   r   Zassert_produces_warningFutureWarningr   r   r   )r   r   r   r   r   r    test_pickle_pos_args_deprecation  s    r  )`__doc__
__future__r   r   r   datetime	functoolsr   r   r   r   pathlibr   rX   r   r   typingr   r   r   numpyr   r?   Zpandas.compatr   r   Zpandas.compat._optionalr	   Zpandas.compat.compressorsr
   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r   r   r   r   Zpandas._testingZ_testingr   Z-pandas.tests.io.generate_legacy_storage_filesr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   r   r   r   r0   r-   PickleBuffercastr   r   r6   rP   r^   rb   ri   paramr   HIGHEST_PROTOCOLrs   r   r   Zfixturer   r   r   Z
MultiIndexZfrom_arraysr   r   r   r   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr  r  r  r   r   r   r   <module>   s   	$
2	

 


)	



