
     	i                     0   d Z ddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
Zddl
mZ ddlmZ ddlmZ e	j                            d          Ze	j        j        e	j        j        gZe	j                            d	          d
             Zd Zed             ZdS )zd
Tests multithreading behaviour for reading and
parsing files for each parser defined in parsers.py
    )	ExitStack)BytesIO)
ThreadPoolN)	DataFrame)Versionpyarrow_xfailz0ignore:Passing a BlockManager:DeprecationWarningc                 d   | }|j         dk    rlt          j        d          }t          |j                  t          d          k     r3|                    t          j                            d                     dd}fdt          |          D             }t                      5 fd|D             }
                    t          d	                    }|                    |j        |          }|d
         }	|D ]}
t          j        |	|
           	 d d d            d S # 1 swxY w Y   d S )Npyarrowz16.0z+# ValueError: Found non-unique column index)reasond   
   c              3      K   | ]B}d                      d t                    D                                                       V  CdS )
c                 (    g | ]}|d d|d d|d S )d, ).0is     `/srv/django_bis/venv311/lib/python3.11/site-packages/pandas/tests/io/parser/test_multi_thread.py
<listcomp>zBtest_multi_thread_string_io_read_csv.<locals>.<genexpr>.<listcomp>)   s7    FFFAa'''A'''!'''FFF    N)joinrangeencode)r   _max_row_ranges     r   	<genexpr>z7test_multi_thread_string_io_read_csv.<locals>.<genexpr>(   sf         			FF}1E1EFFFGGNNPP     r   c                 T    g | ]$}                     t          |                    %S r   )enter_contextr   )r   bstacks     r   r   z8test_multi_thread_string_io_read_csv.<locals>.<listcomp>/   s-    FFFQ$$WQZZ00FFFr      r   )enginepytestimportorskipr   __version__applymarkermarkxfailr   r   r    r   mapread_csvtmassert_frame_equal)all_parsersrequestparserpa	num_filesbytes_to_dffilespoolresultsfirst_resultresultr   r"   s              @@r   $test_multi_thread_string_io_read_csvr:      s    F}	!! ++2>""WV__44!!)V!WW   MI   y!!  K 
 	8FFFF+FFF"":a==11((6?E22qz 	8 	8F!,7777	8	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8s   )A.D%%D),D)c                 8     fd}fdt                    D             }t                    5 }|                    ||          }ddd           n# 1 swxY w Y   |d         j        }|dd         D ]	}	||	_        
t	          j        |          }
|
S )a  
    Generate a DataFrame via multi-thread.

    Parameters
    ----------
    parser : BaseParser
        The parser object to use for reading the data.
    path : str
        The location of the CSV file to read.
    num_rows : int
        The number of rows to read per task.
    num_tasks : int
        The number of tasks to use for reading this DataFrame.

    Returns
    -------
    df : DataFrame
    c                     | \  }}|s                     dd|dg          S                      ddt          |          dz   |dg          S )aj  
        Create a reader for part of the CSV.

        Parameters
        ----------
        arg : tuple
            A tuple of the following:

            * start : int
                The starting row to start for parsing CSV
            * nrows : int
                The number of rows to read.

        Returns
        -------
        df : DataFrame
        r   date)	index_colheadernrowsparse_datesN   	   )r>   r?   skiprowsr@   rA   )r,   int)argstartr@   r1   paths      r   readerz0_generate_multi_thread_dataframe.<locals>.readerN   sy    $ u 	??!5vh #    ZZ!^  
 
 	
r   c                 *    g | ]}|z  z  z  fS r   r   )r   r   num_rows	num_taskss     r   r   z4_generate_multi_thread_dataframe.<locals>.<listcomp>p   s9       ?@A	"H	$9:  r   )	processesNr   rB   )r   r   r+   columnspdconcat)r1   rH   rK   rL   rI   tasksr6   r7   r?   rfinal_dataframes   ````       r    _generate_multi_thread_dataframerT   :   s   ( 
  
  
  
  
  
D    DI)DTDT  E 
i	(	(	( *D((65))* * * * * * * * * * * * * * * QZFQRR[  		i((Os   AA!Ac                 f   d}d}| }d}t          t          j                            d                              |          t          j                            d                              |          t          j                            d                              |          t          j                            d                              |          t          j                            d                              |          dg|z  dg|z  dg|z  t	          j        d|d	
          t          j        |d          d
          }t          j        |          5 }|	                    |           t          ||||          }t          j        ||           d d d            d S # 1 swxY w Y   d S )N   0   z__thread_pool_reader__.csv   foobarbazz20000101 09:00:00s)periodsfreqint64)dtype)
ar!   cr   erY   rZ   r[   r=   rE   )r   nprandomdefault_rngrO   
date_rangearanger-   ensure_cleanto_csvrT   r.   )r/   rL   rK   r1   	file_namedfrH   rS   s           r   )test_multi_thread_path_multipart_read_csvrm      s    IHF,I	&&q))00::&&q))00::&&q))00::&&q))00::&&q))00::7X%7X%7X%M"5xcRRR9XW555	
 	

 
B 
	#	# 3t
		$:D(I
 
 	b/2223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   =F&&F*-F*)__doc__
contextlibr   ior   multiprocessing.poolr   numpyrd   r%   pandasrO   r   pandas._testing_testingr-   pandas.util.versionr   r)   usefixturesxfail_pyarrow
single_cpuslow
pytestmarkfilterwarningsr:   rT   rm   r   r   r   <module>r}      s2    !                 + + + + + +                      ' ' ' ' ' '''88
 K
K
 NOO8 8 PO8<C C CL 3 3 3 3 3r   