a
    Ofw7                     @  s6  d dl mZ d dlmZ d dlmZ d dlZd dlZd dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZm Z m!Z! erd dl"m#Z#m$Z$m%Z% d dl&m'Z'm(Z(m)Z)m*Z* d dl+m,Z,m-Z- G dd deZ.dddddZ/dddddZ0dS )    )annotations)defaultdict)TYPE_CHECKINGN)libparsers)import_optional_dependency)DtypeWarning)find_stack_levelpandas_dtype)concat_compatunion_categoricals)CategoricalDtype)ensure_index_from_sequences)dedup_namesis_potential_multi_index)
ParserBaseParserErroris_index_col)HashableMappingSequence)	ArrayLikeDtypeArgDtypeObjReadCsvBuffer)Index
MultiIndexc                      s   e Zd ZU ded< ded< ddd fdd	Zdd
ddZdd
ddZddddddZdddddZddddddZ	  Z
S ) CParserWrapperbool
low_memoryzparsers.TextReader_readerzReadCsvBuffer[str]None)srcreturnc                   s  t  | || _| }|dd| _| jdu|d< | j|d< | jj	|d< dD ]}||d  qPt
|dd |d< d|vs|d tju rd	|d< |d d
krtd
 tj|fi || _| jj| _| jd u }| jjd u rd | _n"| | jj| j|\| _| _| _}| jd u r$tt| jj| _| jd d  | _| jr| | j| j | jd us\J | jdkrt | js|   | j t!| jt! kr fddt"| jD | _t!| jt! k r|   | j | #| j | $  | j| _| j%s~| jj&dkrHt'| jrHd| _(| )| j| j\}| _| _| jd u rH|| _| jjd u r~|s~| jd uslJ d gt!| j | _| jj&dk| _*d S )Nr    FZallow_leading_colsusecolson_bad_lines)Zstorage_optionsencodingZ
memory_mapcompressiondtypeZdtype_backendnumpyZpyarrowstringc                   s$   g | ]\}}| v s| v r|qS  r,   ).0inr%   r,   [/var/www/ai-form-bot/venv/lib/python3.9/site-packages/pandas/io/parsers/c_parser_wrapper.py
<listcomp>   s   z+CParserWrapper.__init__.<locals>.<listcomp>r   T)+super__init__kwdscopypopr    	index_colr%   r&   valueensure_dtype_objsgetr   Z
no_defaultr   r   Z
TextReaderr!   Zunnamed_colsnamesheaderZ_extract_multi_indexer_columnsindex_names	col_nameslistrangetable_width
orig_names_evaluate_usecolsZusecols_dtypesetissubsetZ_validate_usecols_nameslen	enumerateZ_validate_parse_dates_presence_set_noconvert_columns_has_complex_date_colleading_colsr   Z_name_processedZ_clean_index_namesZ_implicit_index)selfr#   r5   keyZpassed_namesr>   	__class__r0   r1   r4   <   s    


	
zCParserWrapper.__init__)r$   c                 C  s&   z| j   W n ty    Y n0 d S N)r!   close
ValueError)rL   r,   r,   r1   rQ      s    zCParserWrapper.closec                   s^   | j dusJ dd t| j D   fdd| jD }| || j}|D ]}| j| qHdS )z
        Set the columns that should not undergo dtype conversions.

        Currently, any column that is involved with date parsing will not
        undergo such conversions.
        Nc                 S  s   i | ]\}}||qS r,   r,   )r-   r.   xr,   r,   r1   
<dictcomp>       z9CParserWrapper._set_noconvert_columns.<locals>.<dictcomp>c                   s   g | ]} | qS r,   r,   r-   rS   Z
names_dictr,   r1   r2      rU   z9CParserWrapper._set_noconvert_columns.<locals>.<listcomp>)rC   rH   r<   Z_set_noconvert_dtype_columnsr!   Zset_noconvert)rL   Zcol_indicesZnoconvert_columnscolr,   rW   r1   rI      s    z%CParserWrapper._set_noconvert_columnsNz
int | Nonez_tuple[Index | MultiIndex | None, Sequence[Hashable] | MultiIndex, Mapping[Hashable, ArrayLike]])nrowsr$   c                   s  z,| j r| j|}t|}n| j|}W n ty   | jrd| _t| jt	| j| j
}| j|| jd\} }|  | j | jd ur|    fdd| D }| |f Y S |    Y n0 d| _| j}| jjr| jrtdg }| j
r,| jjt| j
kr,tdt| j
 d| jj dt| jjD ]H}| j
d u rT||}	n|| j
| }	| j|	|d	d
}	||	 q8t|}| jd ur| |}t|t	|| j
}t| }
dd t||
D }|  ||\}}| || j}nt| }
| jd usJ t!| j}t|t	|| j
}| jd ur@| |}dd |
D }| jd u rf| "|| dd t||
D }|  ||\}}| #|||\}}|||fS )NFr)   c                   s   i | ]\}}| v r||qS r,   r,   )r-   kvcolumnsr,   r1   rT      rU   z'CParserWrapper.read.<locals>.<dictcomp>z file structure not yet supportedz,Could not construct index. Requested to use z number of columns, but z left to parse.T)try_parse_datesc                 S  s   i | ]\}\}}||qS r,   r,   r-   r[   r.   r\   r,   r,   r1   rT   .  rU   c                 S  s   g | ]}|d  qS )   r,   rV   r,   r,   r1   r2   F  rU   z'CParserWrapper.read.<locals>.<listcomp>c                 S  s   i | ]\}\}}||qS r,   r,   r`   r,   r,   r1   rT   J  rU   )$r    r!   Zread_low_memory_concatenate_chunksreadStopIterationZ_first_chunkr   rC   r   r8   Z_get_empty_metar)   Z_maybe_make_multi_index_columnsr?   r%   _filter_usecolsitemsrQ   r<   rK   rJ   NotImplementedErrorrG   r   rA   r7   _maybe_parse_datesappendr   sortedzipZ_do_date_conversionsr@   Z_check_data_lengthZ_make_index)rL   rY   chunksdatar<   indexZcol_dictZarraysr.   valuesZ	data_tupsZcolumn_namesZ	date_dataZalldatar,   r]   r1   rc      s    







zCParserWrapper.readzSequence[Hashable])r<   r$   c                   s@   |  | j|  d ur<t|t kr< fddt|D }|S )Nc                   s$   g | ]\}}| v s| v r|qS r,   r,   )r-   r.   namer0   r,   r1   r2   U  s   z2CParserWrapper._filter_usecols.<locals>.<listcomp>)rD   r%   rG   rH   )rL   r<   r,   r0   r1   re   Q  s    
zCParserWrapper._filter_usecolsTint)rn   r_   c                 C  s4   |r0|  |r0| j|| jd ur(| j| nd d}|S )N)rX   )Z_should_parse_datesZ
_date_convr>   )rL   ro   rn   r_   r,   r,   r1   rh   Z  s    z!CParserWrapper._maybe_parse_dates)N)T)__name__
__module____qualname____annotations__r4   rQ   rI   rc   re   rh   __classcell__r,   r,   rN   r1   r   8   s   
  s	r   zlist[dict[int, ArrayLike]]dict)rl   r$   c           
        s   t | d  }g }i }|D ]  fdd| D }dd |D }dd |D }| }t|trrt|dd| < qt|| < t|d	kr|  jt	t
kr|t  q|rd
|}dd| dg}	tj|	tt d |S )z
    Concatenate chunks of data read with low_memory=True.

    The tricky part is handling Categoricals, where different chunks
    may have different inferred categories.
    r   c                   s   g | ]}|  qS r,   )r7   )r-   chunkrp   r,   r1   r2   o  rU   z'_concatenate_chunks.<locals>.<listcomp>c                 S  s   h | ]
}|j qS r,   rZ   )r-   ar,   r,   r1   	<setcomp>q  rU   z&_concatenate_chunks.<locals>.<setcomp>c                 S  s   h | ]}t |ts|qS r,   )
isinstancer   rV   r,   r,   r1   r{   r  rU   F)Zsort_categoriesra   , z	Columns (zK) have mixed types. Specify dtype option on import or set low_memory=False.)
stacklevel)r@   keysr7   r|   r   r   r   rG   r)   npobjectri   strjoinwarningswarnr   r	   )
rl   r<   Zwarning_columnsresultZarrsZdtypesZnon_cat_dtypesr)   Zwarning_namesZwarning_messager,   ry   r1   rb   c  s*    
 

rb   z*DtypeArg | dict[Hashable, DtypeArg] | Nonez*DtypeObj | dict[Hashable, DtypeObj] | None)r)   r$   c                   sx   t trHt  t fdd} D ]}t| ||< q.|S t trdfddD S durttS S )zc
    Ensure we have either None, a dtype object, or a dictionary mapping to
    dtype objects.
    c                     s    S rP   r,   r,   )default_dtyper,   r1   <lambda>  rU   z#ensure_dtype_objs.<locals>.<lambda>c                   s   i | ]}|t  | qS r,   r
   )r-   r[   rZ   r,   r1   rT     rU   z%ensure_dtype_objs.<locals>.<dictcomp>N)r|   r   r   default_factoryr   rw   )r)   Zdtype_convertedrM   r,   )r   r)   r1   r:     s    

r:   )1
__future__r   collectionsr   typingr   r   r*   r   Zpandas._libsr   r   Zpandas.compat._optionalr   Zpandas.errorsr   Zpandas.util._exceptionsr	   Zpandas.core.dtypes.commonr   Zpandas.core.dtypes.concatr   r   Zpandas.core.dtypes.dtypesr   Zpandas.core.indexes.apir   Zpandas.io.commonr   r   Zpandas.io.parsers.base_parserr   r   r   collections.abcr   r   r   Zpandas._typingr   r   r   r   Zpandasr   r   r   rb   r:   r,   r,   r,   r1   <module>   s.     -%