1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 import Queue
26 import thread, threading
27 from threading import Thread, Condition
28 import sys
29 import time
30 import traceback
31 import warn
32
33 warn = warn.Warn("threadutils.ThreadPool")
34
35 try:
36 True
37 except:
38 __builtins__['True'] = 1
39 __builtins__['False'] = 0
40
44
46 if not self.__closed:
47 self.stop()
48
50 Thread.__init__(self)
51 self.cond=Condition()
52 self.putfunction = pool.put
53 self.exit = False
54 self.start()
55 self.site = None
56 self.exception = None
57 self.pool = pool
58 self.__closed = False
59
60 - def prepare(self, putfunction, endnotify_function, function, args, argd):
61 self.function = function
62 self.args = args
63 self.argd = argd
64 self.exception = None
65 self.putfunction = putfunction
66 self.endnotify = endnotify_function
67 self.cond.acquire()
68 self.cond.notify()
69 self.cond.release()
70
72 self.cond.acquire()
73 self.exit = True
74 self.cond.notify()
75 self.cond.release()
76 self.__closed = True
77
79 while True:
80 self.cond.acquire()
81 self.putfunction(self)
82 if not self.exit: self.cond.wait()
83 self.cond.release()
84 if self.exit: break
85 exc, res = None, None
86 try:
87 res=self.function(*self.args,**self.argd)
88 except Exception, e:
89 warn.warn("Exception in ThreadPool: %s\n"%(e),
90 self.args, self.argd)
91 exc = e
92 if self.endnotify:
93 if callable(self.endnotify):
94 self.endnotify(res, exc)
95 else:
96 self.endnotify[0](res, exc, self.endnotify)
97
99 - def _init_(self, maxthreads, can_grow):
100 self.maxthreads = maxthreads
101 self.cond = Condition()
102 self.wait = False
103 self.cnt = self.maxthreads
104 self.can_grow = can_grow
105
107 """ Returns the thread |obj| to the pool, internal method """
108 self.cond.acquire()
109
110 self.put(obj)
111
112 if self.wait and self.cnt == 0:
113 self.cond.notify()
114 self.wait = False
115 self.cond.release()
116
117 - def begin(self, cnt=None):
118 """ Begin a synchronous block. Block's inside |cnt| threads are
119 executed, but the program control will wait it
120 in the block (in the self.end call). If |cnt| is not present,
121 self.maxthreads is used"""
122 self.cond.acquire()
123 if cnt:
124 self.cnt = cnt
125 else:
126 self.cnt = self.maxthreads
127 self.wait = True
128 self.cond.release()
129
131 """ Wait to finish all the threads in this pool, since the
132 self.begin call """
133 self.cond.acquire()
134 if self.wait: self.cond.wait()
135 self.cond.release()
136
137 - def execute(self, endfunction, function, *args, **argd):
138 """ Execute |function| into a diferent thread with args (positional
139 arguments) and argd (named arguments).
140 If |endfunction| is diferent of None:
141 If it is callable: It's called with two arguments. The
142 first argument is the result, the second is the
143 exceptionCatched (if any).
144 In other words: endfunction(res, exc)
145
146 If |endfunction| is not callable: the first item is called as a
147 function and |endfunction| is passed as third argument. First
148 two arguments are equal to the previous case.
149 In other words: endfunction[0](res, exc, endfunction)
150 |result| is
151 the result value of calling |function|, and |exceptionCatched| is not
152 None if an exception was catched while |function| was running """
153 if self.can_grow:
154 try:
155 runthread = self.get_nowait()
156 except Queue.Empty:
157 runthread = RunThread(self)
158 runthread = self.get()
159 else:
160
161 runthread=self.get()
162
163 if self.wait:
164 self.cond.acquire()
165 self.cnt = self.cnt - 1
166 self.cond.release()
167 runthread.prepare(self.put_block,endfunction,function,args,argd)
168 else:
169 runthread.prepare(self.put, endfunction, function, args, argd)
170
173
174
176 """ Create at most |maxthreads| to retrieve sites. It must be used with
177 Queue |get| (get a ThreadFetch) and |put| (return the thread) methods. """
180
181 - def __init__(self, maxthreads, can_grow = False):
182 """ Create a pool of threads """
183
184 if can_grow:
185 Queue.Queue.__init__(self)
186 else:
187 Queue.Queue.__init__(self, maxthreads)
188 self.__closed = False
189 self._init_(maxthreads, can_grow)
190 for i in range(0, maxthreads): RunThread(self)
191
193 if not self.__closed: self.close()
194
196 """ Close the ThreadPool object"""
197 self.__closed = True
198 for i in range(0, self.maxthreads):
199 thr = self.get()
200 thr.stop()
201
203 """ A Fake ThreadPool that share Thread objects (RunThread)
204 between ThreadPool's objects. Useful to save resources """
205 - def __init__(self, maxthreads, queue, can_grow = False):
206 self._init_(maxthreads, can_grow)
207 self.queue = queue
208 self.get = self.queue.get
209 self.put = self.queue.put
210