AlkantarClanX12

Your IP : 18.217.118.7

Current Path : /opt/alt/python35/lib64/python3.5/multiprocessing/__pycache__/
Upload File :
Current File : //opt/alt/python35/lib64/python3.5/multiprocessing/__pycache__/pool.cpython-35.opt-1.pyc



��Yf�d�@s�ddgZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddlm
Z
mZdZdZ
dZej�Zdd	�Zd
d�ZGdd
�d
e�ZGdd�d�Zdd�ZGdd�de�Zdfdddd�Zdd�ZGdd�de�ZGdd�de�ZeZGdd�de�ZGdd�de�ZGd d!�d!e�Z Gd"d�de�Z!dS)#�Pool�
ThreadPool�N�)�util)�get_context�TimeoutError�cCstt|��S)N)�list�map)�args�r�9/opt/alt/python35/lib64/python3.5/multiprocessing/pool.py�mapstar+srcCsttj|d|d��S)Nrr)r	�	itertools�starmap)rrrr
�starmapstar.src@s(eZdZdd�Zdd�ZdS)�RemoteTracebackcCs
||_dS)N)�tb)�selfrrrr
�__init__6szRemoteTraceback.__init__cCs|jS)N)r)rrrr
�__str__8szRemoteTraceback.__str__N)�__name__�
__module__�__qualname__rrrrrr
r5src@s(eZdZdd�Zdd�ZdS)�ExceptionWithTracebackcCsDtjt|�||�}dj|�}||_d||_dS)N�z

"""
%s""")�	traceback�format_exception�type�join�excr)rr rrrr
r<s	zExceptionWithTraceback.__init__cCst|j|jffS)N)�rebuild_excr r)rrrr
�
__reduce__Asz!ExceptionWithTraceback.__reduce__N)rrrrr"rrrr
r;srcCst|�|_|S)N)r�	__cause__)r rrrr
r!Dsr!cs@eZdZdZ�fdd�Zdd�Zdd�Z�S)�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.csAt|�|_t|�|_tt|�j|j|j�dS)N)�reprr �value�superr$r)rr r&)�	__class__rr
rPszMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r&r )rrrr
rUs	zMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r(r)rrrr
�__repr__YszMaybeEncodingError.__repr__)rrr�__doc__rrr)rr)r(r
r$Lsr$Fc'Cs|j}|j}t|d�r;|jj�|jj�|dk	rQ||�d}x�|dksx|r�||kr�y
|�}	Wn&ttfk
r�tj	d�PYnX|	dkr�tj	d�P|	\}
}}}
}yd||
|�f}WnUt
k
rM}z5|r/|tk	r/t||j
�}d|f}WYdd}~XnXy||
||f�Wnbt
k
r�}zBt||d�}tj	d|�||
|d|ff�WYdd}~XnXd}	}
}}}
}|d7}qZWtj	d	|�dS)
N�_writerrz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFrz0Possible encoding error while sending result: %szworker exiting after %d tasks)�put�get�hasattrr+�close�_reader�EOFError�OSErrorr�debug�	Exception�_helper_reraises_exceptionr�
__traceback__r$)�inqueue�outqueue�initializer�initargsZmaxtasksZwrap_exceptionr,r-Z	completed�task�job�i�funcr�kwds�result�e�wrappedrrr
�worker]sD		


!


	,rCcCs
|�dS)z@Pickle-able helper function for use by _guarded_task_generation.Nr)Zexrrr
r5�sr5c@s�eZdZdZdZdd�Zddfdddd�Zdd	�Zd
d�Zdd
�Z	dd�Z
fidd�Zddd�Zddd�Z
ddddd�Zdd�Zddd�Zddd�Zfidddd �Zdddd!d"�Zdddd#d$�Zed%d&��Zed'd(��Zed)d*��Zed+d,��Zd-d.�Zd/d0�Zd1d2�Zd3d4�Zed5d6��Zed7d8��Z d9d:�Z!d;d<�Z"dS)=rzS
    Class which supports an async version of applying functions to arguments.
    TcOs|jj||�S)N)�_ctx�Process)rrr?rrr
rE�szPool.ProcessNcCs#|pt�|_|j�tj�|_i|_t|_||_	||_
||_|dkrvtj
�psd}|dkr�td��|dk	r�t|�r�td��||_g|_|j�tjdtjd|f�|_d|j_t|j_|jj�tjdtjd|j|j|j|j|jf�|_d|j_t|j_|jj�tjdtjd|j|j |jf�|_!d|j!_t|j!_|j!j�t"j#||j$d|j|j%|j|j|j|j|j!|jfdd�|_&dS)	Nrz&Number of processes must be at least 1zinitializer must be a callable�targetrTZexitpriority�)'rrD�
_setup_queues�queue�Queue�
_taskqueue�_cache�RUN�_state�_maxtasksperchild�_initializer�	_initargs�os�	cpu_count�
ValueError�callable�	TypeError�
_processes�_pool�_repopulate_pool�	threadingZThreadr�_handle_workers�_worker_handler�daemon�start�
_handle_tasks�
_quick_put�	_outqueue�
_task_handler�_handle_results�
_quick_get�_result_handlerrZFinalize�_terminate_pool�_inqueue�
_terminate)r�	processesr9r:Zmaxtasksperchild�contextrrr
r�sT
							
		
		
		
z
Pool.__init__cCswd}xjttt|j���D]M}|j|}|jdk	r"tjd|�|j�d}|j|=q"W|S)z�Cleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        FNzcleaning up worker %dT)�reversed�range�lenrX�exitcoderr3r)rZcleanedr=rCrrr
�_join_exited_workers�s"

zPool._join_exited_workerscCs�x�t|jt|j��D]�}|jdtd|j|j|j|j	|j
|jf�}|jj|�|j
jdd�|_
d|_|j�tjd�qWdS)z�Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        rFrrEZ
PoolWorkerTzadded workerN)rlrWrmrXrErCrgrarPrQrO�_wrap_exception�append�name�replacer]r^rr3)rr=�wrrr
rY�s#	
zPool._repopulate_poolcCs|j�r|j�dS)zEClean up any exited workers and start replacements for them.
        N)rorY)rrrr
�_maintain_pool�szPool._maintain_poolcCsL|jj�|_|jj�|_|jjj|_|jjj|_	dS)N)
rDZSimpleQueuergrar+�sendr`r0�recvrd)rrrr
rH�szPool._setup_queuescCs|j|||�j�S)z6
        Equivalent of `func(*args, **kwds)`.
        )�apply_asyncr-)rr>rr?rrr
�apply�sz
Pool.applycCs|j||t|�j�S)zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        )�
_map_asyncrr-)rr>�iterable�	chunksizerrr
r
szPool.mapcCs|j||t|�j�S)z�
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        )rzrr-)rr>r{r|rrr
rszPool.starmapcCs|j||t|||�S)z=
        Asynchronous version of `starmap()` method.
        )rzr)rr>r{r|�callback�error_callbackrrr
�
starmap_asyncszPool.starmap_asyncccs�y>d}x1t|�D]#\}}||||fifVqWWn@tk
r�}z ||dt|fifVWYdd}~XnXdS)z�Provides a generator of tasks for imap and imap_unordered with
        appropriate handling for iterables which throw exceptions during
        iteration.rN���)�	enumerater4r5)rZ
result_jobr>r{r=�xrArrr
�_guarded_task_generationszPool._guarded_task_generationrcCs�|jtkrtd��|dkret|j�}|jj|j|j||�|j	f�|St
j|||�}t|j�}|jj|j|jt|�|j	f�dd�|D�SdS)zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        zPool not runningrcss"|]}|D]}|Vq
qdS)Nr)�.0�chunk�itemrrr
�	<genexpr>@szPool.imap.<locals>.<genexpr>N)
rNrMrT�IMapIteratorrLrKr,r��_job�_set_lengthr�
_get_tasksr)rr>r{r|r@�task_batchesrrr
�imap's 	
	
z	Pool.imapcCs�|jtkrtd��|dkret|j�}|jj|j|j||�|j	f�|St
j|||�}t|j�}|jj|j|jt|�|j	f�dd�|D�SdS)zL
        Like `imap()` method but ordering of results is arbitrary.
        zPool not runningrcss"|]}|D]}|Vq
qdS)Nr)r�r�r�rrr
r�[sz&Pool.imap_unordered.<locals>.<genexpr>N)
rNrMrT�IMapUnorderedIteratorrLrKr,r�r�r�rr�r)rr>r{r|r@r�rrr
�imap_unorderedBs 	
	
zPool.imap_unorderedcCs_|jtkrtd��t|j||�}|jj|jd|||fgdf�|S)z;
        Asynchronous version of `apply()` method.
        zPool not runningrN)rNrMrT�ApplyResultrLrKr,r�)rr>rr?r}r~r@rrr
rx]s
+zPool.apply_asynccCs|j||t|||�S)z9
        Asynchronous version of `map()` method.
        )rzr)rr>r{r|r}r~rrr
�	map_asynchszPool.map_asyncc
Cs�|jtkrtd��t|d�s6t|�}|dkrztt|�t|j�d�\}}|rz|d7}t|�dkr�d}tj	|||�}t
|j|t|�|d|�}	|jj
|j|	j||�df�|	S)zY
        Helper function to implement map, starmap and their async counterparts.
        zPool not running�__len__N�rrr~)rNrMrTr.r	�divmodrmrXrr��	MapResultrLrKr,r�r�)
rr>r{Zmapperr|r}r~Zextrar�r@rrr
rzps&(
		
zPool._map_asynccCsrtj�}xB|jtks6|jrP|jtkrP|j�tjd�qW|j	j
d�tjd�dS)Ng�������?zworker handler exiting)
rZ�current_threadrNrMrL�	TERMINATEru�time�sleeprKr,rr3)�pool�threadrrr
r[�s*
zPool._handle_workersc
Cs�tj�}x+t|jd�D]
\}}d}z�x�|D]�}|jrXtjd�Py||�Wq;tk
r�}	zN|dd�\}
}y||
j|d|	f�Wnt	k
r�YnXWYdd}	~	Xq;Xq;W|rtjd�|r�|dnd}||d�wPWdd}}}
XqWtjd�yFtjd�|j
d�tjd�x|D]}|d�qkWWntk
r�tjd	�YnXtjd
�dS)Nz'task handler found thread._state != RUNrFzdoing set_length()rztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr�)rZr��iterr-rNrr3r4�_set�KeyErrorr,r2)
�	taskqueuer,r8r��cacher�ZtaskseqZ
set_lengthr;rAr<�idx�prrr
r_�sB
	








zPool._handle_taskscCstj�}x�y
|�}Wn)ttfk
rGtjd�dSYnX|jr_tjd�P|dkrytjd�P|\}}}y||j||�Wntk
r�YnXd}}}qWx�|r�|jt	kr�y
|�}Wn)ttfk
rtjd�dSYnX|dkr4tjd�q�|\}}}y||j||�Wntk
roYnXd}}}q�Wt
|d�r�tjd�y2x+td�D]}|jj
�s�P|�q�WWnttfk
r�YnXtjdt|�|j�dS)	Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr0z"ensuring that outqueue is not full�
z7result handler exiting: len(cache)=%s, thread._state=%s)rZr�r2r1rr3rNr�r�r�r.rlr0�pollrm)r8r-r�r�r;r<r=�objrrr
rc�sZ

		




	


	zPool._handle_resultsccsDt|�}x1ttj||��}|s1dS||fVqWdS)N)r��tupler�islice)r>�it�sizer�rrr
r�szPool._get_taskscCstd��dS)Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)rrrr
r"szPool.__reduce__cCs5tjd�|jtkr1t|_t|j_dS)Nzclosing pool)rr3rNrM�CLOSEr\)rrrr
r/s
	z
Pool.closecCs0tjd�t|_t|j_|j�dS)Nzterminating pool)rr3r�rNr\rh)rrrr
�	terminates
	zPool.terminatecCsVtjd�|jj�|jj�|jj�x|jD]}|j�q>WdS)Nzjoining pool)rr3r\rrbrerX)rr�rrr
rs



z	Pool.joincCsZtjd�|jj�x9|j�rU|jj�rU|jj�tj	d�qWdS)Nz7removing tasks from inqueue until task handler finishedr)
rr3Z_rlock�acquire�is_aliver0r�rwr�r�)r7�task_handlerr�rrr
�_help_stuff_finish(s



zPool._help_stuff_finishc	
Cs�tjd�t|_t|_tjd�|j||t|��t|_|jd�tjd�tj�|k	r�|j	�|r�t
|dd�r�tjd�x'|D]}	|	jdkr�|	j�q�Wtjd�tj�|k	r�|j	�tjd�tj�|k	r&|j	�|r�t
|dd�r�tjd	�x8|D]0}	|	j
�rStjd
|	j�|	j	�qSWdS)Nzfinalizing poolz&helping task handler/workers to finishzjoining worker handlerrr�zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)rr3r�rNr�rmr,rZr�rr.rnr�r��pid)
�clsr�r7r8r�Zworker_handlerr�Zresult_handlerr�r�rrr
rf1s6
		
	










zPool._terminate_poolcCs|S)Nr)rrrr
�	__enter___szPool.__enter__cCs|j�dS)N)r�)r�exc_typeZexc_valZexc_tbrrr
�__exit__bsz
Pool.__exit__)#rrrr*rprErrorYrurHryr
rrr�r�r�rxr�rz�staticmethodr[r_rcr�r"r/r�rr��classmethodrfr�r�rrrr
r�sF	8	

.<			.c@s^eZdZdd�Zdd�Zdd�Zddd	�Zdd
d�Zdd
�ZdS)r�cCsJtj�|_tt�|_||_||_||_|||j<dS)N)	rZZEvent�_event�next�job_counterr�rL�	_callback�_error_callback)rr�r}r~rrr
rks			zApplyResult.__init__cCs
|jj�S)N)r�Zis_set)rrrr
�readysszApplyResult.readycCs|jS)N)�_success)rrrr
�
successfulvszApplyResult.successfulNcCs|jj|�dS)N)r��wait)r�timeoutrrr
r�zszApplyResult.waitcCs<|j|�|j�st�|jr/|jS|j�dS)N)r�r�rr��_value)rr�rrr
r-}s
	zApplyResult.getcCsu|\|_|_|jr4|jr4|j|j�|jrW|jrW|j|j�|jj�|j|j=dS)N)r�r�r�r�r��setrLr�)rr=r�rrr
r��s
zApplyResult._set)	rrrrr�r�r�r-r�rrrr
r�is	r�c@s(eZdZdd�Zdd�ZdS)r�cCs�tj|||d|�d|_dg||_||_|dkrjd|_|jj�||j=n||t	||�|_dS)Nr~Tr)
r�rr�r��
_chunksize�_number_leftr�r�r��bool)rr�r|�lengthr}r~rrr
r�s			

zMapResult.__init__cCs�|\}}|r�||j||j|d|j�<|jd8_|jdkr�|jrn|j|j�|j|j=|jj�nEd|_||_|j	r�|j	|j�|j|j=|jj�dS)NrrF)
r�r�r�r�rLr�r�r�r�r�)rr=Zsuccess_result�successr@rrr
r��s%	
			
zMapResult._setN)rrrrr�rrrr
r��s
r�c@sUeZdZdd�Zdd�Zddd�ZeZdd	�Zd
d�ZdS)r�cCsktjtj��|_tt�|_||_tj	�|_
d|_d|_i|_
|||j<dS)Nr)rZZ	ConditionZLock�_condr�r�r�rL�collections�deque�_items�_index�_length�	_unsorted)rr�rrr
r�s				zIMapIterator.__init__cCs|S)Nr)rrrr
�__iter__�szIMapIterator.__iter__NcCs�|j��y|jj�}Wn�tk
r�|j|jkrEt�|jj|�y|jj�}Wn0tk
r�|j|jkr�t�t�YnXYnXWdQRX|\}}|r�|S|�dS)N)	r�r��popleft�
IndexErrorr�r��
StopIterationr�r)rr�r�r�r&rrr
r��s"


zIMapIterator.nextc
Cs�|j��|j|kr�|jj|�|jd7_xJ|j|jkr�|jj|j�}|jj|�|jd7_q;W|jj�n
||j|<|j|jkr�|j|j	=WdQRXdS)Nr)
r�r�r�rqr��pop�notifyr�rLr�)rr=r�rrr
r��s

zIMapIterator._setc	CsJ|j�:||_|j|jkr?|jj�|j|j=WdQRXdS)N)r�r�r�r�rLr�)rr�rrr
r��s

	
zIMapIterator._set_length)	rrrrr�r��__next__r�r�rrrr
r��s
r�c@seZdZdd�ZdS)r�c
Cs`|j�P|jj|�|jd7_|jj�|j|jkrU|j|j=WdQRXdS)Nr)r�r�rqr�r�r�rLr�)rr=r�rrr
r��s

zIMapUnorderedIterator._setN)rrrr�rrrr
r��sr�c@s[eZdZdZedd��Zddfdd�Zdd�Zed	d
��ZdS)rFcOsddlm}|||�S)Nr)rE)ZdummyrE)rr?rErrr
rEszThreadPool.ProcessNcCstj||||�dS)N)rr)rrir9r:rrr
rszThreadPool.__init__cCs@tj�|_tj�|_|jj|_|jj|_dS)N)rIrJrgrar,r`r-rd)rrrr
rHszThreadPool._setup_queuesc
CsF|j�6|jj�|jjdg|�|jj�WdQRXdS)N)Z	not_emptyrI�clear�extendZ
notify_all)r7r�r�rrr
r�s

zThreadPool._help_stuff_finish)	rrrrpr�rErrHr�rrrr
rs
)"�__all__rZrIrr�rRr�rrrrrrMr�r��countr�rrr4rrr!r$rCr5�objectrr�ZAsyncResultr�r�r�rrrrr
�<module>
s<		*��&%@