-
Notifications
You must be signed in to change notification settings - Fork 19
/
Copy pathDemo.py
61 lines (45 loc) · 1.24 KB
/
Demo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# -*- encoding:utf-8 -*-
"""
使用sklearn的parallel作为demo
"""
import numpy as np
from ProcessMonitor import add_process_wrapper
from sklearn.externals.joblib import Parallel
from sklearn.externals.joblib import delayed
import time
from concurrent.futures import ProcessPoolExecutor
__author__ = 'BBFamily'
"""
只需要在具体process job @add_procee_wrapper就ok了
"""
@add_process_wrapper
def do_process_job(jb):
c = count(jb)
end_tick = 100
for cb in c:
time.sleep(1)
if cb % 10 == 0:
print(cb)
if end_tick < cb:
break
return jb
def when_done(r):
print('job {} done:'.format(r.result()))
def make_parallel_poll_jobs():
with ProcessPoolExecutor() as pool:
n_jobs = 10
for jb in np.arange(0, n_jobs):
future_result = pool.submit(do_process_job, jb)
future_result.add_done_callback(when_done)
def make_parallel_jobs():
n_jobs = 10
parallel = Parallel(
n_jobs=n_jobs, verbose=0, pre_dispatch='2*n_jobs')
out = parallel(delayed(do_process_job)(jb) for jb in np.arange(0, n_jobs))
return out
def count(n):
while True:
yield n
n += 1
if __name__ == "__main__":
make_parallel_jobs()