'Python multiprocessing not calling worker method, but the script itself
I am trying to leverage all my cpu cores for doing multiprocessing, and therefore trying to use multiprocessing module of python. I have tried so many code samples on internet but all of them calling the script itself rather than the worker method inside script. I am on windows 7.
I have tried apply, apply_async, pool, process , all these methods.
Below is code samples (little clumsy as i tried many ways). Also i am running script directly from prompt as python multiproc.py
Please note that all these end up printing following, which shows the script being called. But none of these called print method inside worker function or even the file create function (in case print may not work in multiprocessing):
66----
66----
66----
Below are different code samples I tried (separated by ----):
import sys
import multiprocessing as mp
from multiprocessing import Pool, TimeoutError
import time
import os
# from multiproc_child import worker1
# import multiproc_child
print("666---------------")
import multiprocessing
from os import getpid
def worker(procnum):
print('I am number %d in process %d' % (procnum, getpid()))
pid= getpid()
with open(r"d:\temp\p\\"+str(pid), "w") as fw:
fw.write("worker")
return pid
if __name__ == '__main__':
pool = multiprocessing.Pool(processes = 3)
print(pool.map(worker, range(5)))
sys.exit(66)
print("555-----------------------")
def worker():
"""worker function"""
print ('Worker', flush=True)
return 2
if __name__ == '__main__':
jobs = []
for i in range(5):
p1 = mp.Process(target=worker)
jobs.append(p1)
p2 = mp.Process(target=worker)
jobs.append(p2)
print(p1.start())
print(p2.start())
print(p1.join())
print(p2.join())
print("exitcode:", p1.exitcode, p1.name)
print("after join")
sys.exit(55)
print("4444--------------------------")
def cube(x):
print("cube...",flush=True)
return(x**3)
if __name__ == '__main__':
pool = Pool(processes=4)
cube(2)
# results = [pool.apply_async(cube, args=(x,)) for x in range(1,7)]
results = []
# results.append(pool.apply_async(cube, args=(5)))
# results.append(pool.apply_async(cube, args=(6)))
results.append(pool.apply(cube, [5,]))
time.sleep(1)
results.append(pool.apply(cube, args=(6)))
time.sleep(1)
print(results)
# pool.join()
print(results[0].get())
print(results[1].get())
# output = [p.get() for p in results]
# print(output)
sys.exit(22)
print("000------------------")
def f1(x):
print("f1",flush=True)
return x*x
if __name__ == '__main__':
# start 4 worker processes
with Pool(processes=4) as pool:
# print "[0, 1, 4,..., 81]"
print(pool.map(f1, range(10)))
# print same numbers in arbitrary order
for i in pool.imap_unordered(f1, range(10)):
print(i)
# evaluate "f(20)" asynchronously
res = pool.apply_async(f1, (20,)) # runs in *only* one process
print(res.get(timeout=1)) # prints "400"
sys.exit(99)
print("111------------------")
import multiprocessing
def f(x):
print("returnig -1", flush=True)
return -1
n = 0
for i in range(6):
n = max(n, multiprocessing.cpu_count())
return n
if __name__ == '__main__':
print("Main: of range p.map")
with multiprocessing.Pool(4) as p:
# for i in range(6):
print (p.map(f, [1,2,3,4,5]))
sys.exit(11)
print("222------------------")
import time
from timeit import default_timer as timer
from multiprocessing import Pool, cpu_count
def square(n):
time.sleep(2)
print("in square {n}")
return n * n
def main():
start = timer()
print(f'starting computations on {cpu_count()} cores')
values = (2, 4, 6, 8)
with Pool() as pool:
res = pool.map(square, values)
print(res)
end = timer()
print(f'elapsed time: {end - start}')
if __name__ == '__main__':
print("Main: of square")
# main()
print("333------------------")
sys.exit(1)
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
