Package threadutils :: Module ThreadPool'
[hide private]
[frames] | no frames]

Source Code for Module threadutils.ThreadPool'

  1  #coding: iso-8859-1; 
  2  ############################################################################ 
  3  #                                                                          # 
  4  #    This file 'threadutils/ThreadPool.py'                                 # 
  5  #    is part of 'SPyRO: Simple Python Remote Objects'                      # 
  6  #    Copyright (C) 2004-2005 by Eric Sadit Téllez Avila                    # 
  7  #    sadit@lsc.fie.umich.mx or donsadit@gmail.com                          # 
  8  #                                                                          # 
  9  #    This program is free software; you can redistribute it and#or modify  # 
 10  #    it under the terms of the GNU General Public License as published by  # 
 11  #    the Free Software Foundation; either version 2 of the License, or     # 
 12  #    (at your option) any later version.                                   # 
 13  #                                                                          # 
 14  #    This program is distributed in the hope that it will be useful,       # 
 15  #    but WITHOUT ANY WARRANTY; without even the implied warranty of        # 
 16  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         # 
 17  #    GNU General Public License for more details.                          # 
 18  #                                                                          # 
 19  #    You should have received a copy of the GNU General Public License     # 
 20  #    along with this program; if not, write to the                         # 
 21  #    Free Software Foundation, Inc.,                                       # 
 22  #    59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             # 
 23  ############################################################################ 
 24   
 25  import Queue 
 26  import thread, threading 
 27  from threading import Thread, Condition 
 28  import sys 
 29  import time 
 30  import traceback 
 31  import warn 
 32   
 33  warn = warn.Warn("threadutils.ThreadPool") 
 34   
 35  try: 
 36      True 
 37  except: 
 38      __builtins__['True'] = 1 
 39      __builtins__['False'] = 0 
 40   
41 -class RunThread(Thread):
42 - def __getstate__(self):
43 return {}
44
45 - def __del__(self):
46 if not self.__closed: 47 self.stop()
48
49 - def __init__(self, pool):
50 Thread.__init__(self) 51 self.cond=Condition() 52 self.putfunction = pool.put 53 self.exit = False 54 self.start() 55 self.site = None 56 self.exception = None 57 self.pool = pool 58 self.__closed = False
59
60 - def prepare(self, putfunction, endnotify_function, function, args, argd):
61 self.function = function 62 self.args = args 63 self.argd = argd 64 self.exception = None 65 self.putfunction = putfunction 66 self.endnotify = endnotify_function 67 self.cond.acquire() 68 self.cond.notify() 69 self.cond.release()
70
71 - def stop(self):
72 self.cond.acquire() 73 self.exit = True 74 self.cond.notify() 75 self.cond.release() 76 self.__closed = True
77
78 - def run(self):
79 while True: 80 self.cond.acquire() 81 self.putfunction(self) 82 if not self.exit: self.cond.wait() 83 self.cond.release() 84 if self.exit: break 85 exc, res = None, None 86 try: 87 res=self.function(*self.args,**self.argd) 88 except Exception, e: 89 warn.warn("Exception in ThreadPool: %s\n"%(e), 90 self.args, self.argd) 91 exc = e 92 if self.endnotify: 93 if callable(self.endnotify): 94 self.endnotify(res, exc) 95 else: 96 self.endnotify[0](res, exc, self.endnotify)
97
98 -class ThreadPoolMixIn:
99 - def _init_(self, maxthreads, can_grow):
100 self.maxthreads = maxthreads 101 self.cond = Condition() 102 self.wait = False 103 self.cnt = self.maxthreads 104 self.can_grow = can_grow
105
106 - def put_block(self, obj):
107 """ Returns the thread |obj| to the pool, internal method """ 108 self.cond.acquire() 109 #print "AA>", self.qsize(), self.wait 110 self.put(obj) 111 #print "BB>", self.qsize(), self.wait 112 if self.wait and self.cnt == 0: 113 self.cond.notify() 114 self.wait = False 115 self.cond.release()
116
117 - def begin(self, cnt=None):
118 """ Begin a synchronous block. Block's inside |cnt| threads are 119 executed, but the program control will wait it 120 in the block (in the self.end call). If |cnt| is not present, 121 self.maxthreads is used""" 122 self.cond.acquire() 123 if cnt: 124 self.cnt = cnt 125 else: 126 self.cnt = self.maxthreads 127 self.wait = True 128 self.cond.release()
129
130 - def end(self):
131 """ Wait to finish all the threads in this pool, since the 132 self.begin call """ 133 self.cond.acquire() 134 if self.wait: self.cond.wait() 135 self.cond.release()
136
137 - def execute(self, endfunction, function, *args, **argd):
138 """ Execute |function| into a diferent thread with args (positional 139 arguments) and argd (named arguments). 140 If |endfunction| is diferent of None: 141 If it is callable: It's called with two arguments. The 142 first argument is the result, the second is the 143 exceptionCatched (if any). 144 In other words: endfunction(res, exc) 145 146 If |endfunction| is not callable: the first item is called as a 147 function and |endfunction| is passed as third argument. First 148 two arguments are equal to the previous case. 149 In other words: endfunction[0](res, exc, endfunction) 150 |result| is 151 the result value of calling |function|, and |exceptionCatched| is not 152 None if an exception was catched while |function| was running """ 153 if self.can_grow: 154 try: 155 runthread = self.get_nowait() 156 except Queue.Empty: 157 runthread = RunThread(self) 158 runthread = self.get() 159 else: 160 #warn.stack("Trying to get a new runthread") 161 runthread=self.get() 162 #warn.stack("Successful") 163 if self.wait: 164 self.cond.acquire() 165 self.cnt = self.cnt - 1 166 self.cond.release() 167 runthread.prepare(self.put_block,endfunction,function,args,argd) 168 else: 169 runthread.prepare(self.put, endfunction, function, args, argd)
170
171 - def close(self):
172 pass
173 174 #class ThreadPool(Queue.Queue, ThreadPoolMixIn):
175 -class ThreadPool(ThreadPoolMixIn, Queue.Queue):
176 """ Create at most |maxthreads| to retrieve sites. It must be used with 177 Queue |get| (get a ThreadFetch) and |put| (return the thread) methods. """
178 - def __getstate__(self):
179 return {}
180
181 - def __init__(self, maxthreads, can_grow = False):
182 """ Create a pool of threads """ 183 #Queue.Queue.__init__(self, maxthreads) 184 if can_grow: 185 Queue.Queue.__init__(self) 186 else: 187 Queue.Queue.__init__(self, maxthreads) 188 self.__closed = False 189 self._init_(maxthreads, can_grow) 190 for i in range(0, maxthreads): RunThread(self)
191
192 - def __del__(self):
193 if not self.__closed: self.close()
194
195 - def close(self):
196 """ Close the ThreadPool object""" 197 self.__closed = True 198 for i in range(0, self.maxthreads): 199 thr = self.get() 200 thr.stop()
201
202 -class ThreadPoolShared(ThreadPoolMixIn):
203 """ A Fake ThreadPool that share Thread objects (RunThread) 204 between ThreadPool's objects. Useful to save resources """
205 - def __init__(self, maxthreads, queue, can_grow = False):
206 self._init_(maxthreads, can_grow) 207 self.queue = queue 208 self.get = self.queue.get 209 self.put = self.queue.put
210