
    \jD                         d Z ddlZddlZddlZddlmZ ddlmZ  ej                  e
      Z G d de      Z G d de      Zej                  d
d	       Zy)z\Common functionality for concurrent processing. The main entry point is :func:`create_pool`.    N)deque)ThreadPoolExecutorc                       e Zd ZdZddddZy)r   z+Subclass with a lazy consuming imap method.N   )timeoutqueued_tasks_per_workerc             '   $  K   t               | j                  |dz   z  }}|j                  |j                  | j                  c}}fd}	t        | D ])  }
 | ||g|
        t        |      |k(  s! |	        + |r |	        |ryyw)a  Ordered imap that consumes iterables just-in-time.

        References:
            https://gist.github.com/ddelange/c98b05437f80e4b16bf4fc20fde9c999

        Args:
            fn: Function to apply.
            iterables: One (or more) iterable(s) to pass to fn (using zip) as positional argument(s).
            timeout: Per-future result retrieval timeout in seconds.
            queued_tasks_per_worker: Amount of additional items per worker to fetch from iterables to
                    fill the queue: this determines the total queue size.
                Setting 0 will result in a true just-in-time behaviour: when a worker finishes a task,
                    it waits until a result is consumed from the imap generator, at which point next()
                    is called on the input iterable(s) and a new task is submitted.
                Default 2 ensures there is always some work to pick up. Note that at imap startup,
                    the queue will fill up before the first yield occurs.

        Example:
            long_generator = itertools.count()
            with ThreadPoolExecutor(42) as pool:
                result_generator = pool.imap(fn, long_generator)
                for result in result_generator:
                    print(result)
           c                  0             j                        S )z8Block until the next task is done and return the result.)result)popleftr   s   @/root/env/lib/python3.12/site-packages/smart_open/concurrency.pygetz$ThreadPoolExecutor.imap.<locals>.get3   s    9##G,,    N)r   _max_workersr   appendsubmitziplen)selffnr   r   	iterablesfuturesmaxlenr   r   r   argsr   s     `        @r   imapzThreadPoolExecutor.imap   s     2  '4#4#48ORS8S#T")//7>>4;;	- O 	D6"$t$%7|v%e	
 %K s   A0B5BB)__name__
__module____qualname____doc__r    r   r   r   r      s    5+/ &r   r   c                   "    e Zd ZdZd Zd Zd Zy)ConcurrentFuturesPoolz_A class that mimics multiprocessing.pool.Pool but uses concurrent futures instead of processes.c                 &    t        |      | _        y )Nmax_workers)r   executor)r   r&   s     r   __init__zConcurrentFuturesPool.__init__F   s    *{Cr   c              #      K   |D cg c]  }| j                   j                  ||        }}t        j                  j	                  |      D ]  }|j                           y c c}w w)N)r'   r   
concurrentr   as_completedr   )r   functionitemsitemr   futures         r   imap_unorderedz$ConcurrentFuturesPool.imap_unorderedI   s\     DIJD4==''$7JJ ((55g> 	"F--/!	" Ks   A(#A#>A(c                 <    | j                   j                  d       y )NT)wait)r'   shutdown)r   s    r   	terminatezConcurrentFuturesPool.terminateN   s    D)r   N)r   r   r   r    r(   r0   r4   r!   r   r   r#   r#   D   s    iD"
*r   r#   c              #   x   K   t         j                  d|        t        |       }| |j                          y w)Nz0creating concurrent futures pool with %i workersr%   )loggerinfor#   r4   )	processespools     r   create_poolr:   R   s.     
KKBIN Y7D
JNNs   8:)r
   )r    concurrent.futuresr*   
contextlibloggingcollectionsr   r   _ThreadPoolExecutor	getLoggerr   r6   objectr#   contextmanagerr:   r!   r   r   <module>rC      s`    c     H			8	$), )`*F *  r   