'python multiprocessing error along using cupy

Consider simplified example using multiprocessing inside a class that use cupy for simulation. this part

obj = FUNC(par)
obj.simulate()

done in with cupy, then data copy to CPU. multiprocessing does not touch data on GPU but still, I get the following error:

import tqdm
import cupy as cp
import numpy as np
from multiprocessing import Pool


class FUNC:

    def __init__(self, par) -> None:
        self.x = par['x']

    def simulate(self):

        x = self.x
        xs = []

        for t in tqdm.trange(nt):

            dx = cp.random.rand(nn, ns) * 0.001
            x += dx
            xs.append(x.get())

        self.xs = np.asarray(xs)

    def func(self, i):

        x = self.xs[:, :, i]
        return [np.mean(x)]

    def stats(self):

        with Pool(processes=2) as pool:
            data = (pool.map(self.func, range(ns)))

        return cp.asnumpy(data)


if __name__ == "__main__":

    nn = 10
    ns = 10
    nt = 2500

    par = {
        "x": cp.random.randn(nn, ns).astype('f'),
    }

    obj = FUNC(par)
    obj.simulate()
    data = obj.stats()

    print(data.shape)

Process ForkPoolWorker-1:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-2:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-3:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-4:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-5:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error

where am I doing wrong?

Solution 1:^[1]

Adding an answer here to wrap this one up. Didn't stumble upon a Stack Overflow thread when researching this issue so I'm assuming this thread will get more views in the future.

The issue has to do with the default start method not working with CUDA Multiprocessing. By explicitly setting the start method to spawn with multiprocessing.set_start_method('spawn', force=True) this issue is resolved.

Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution	Source
Solution 1	Paul Burkart

'python multiprocessing error along using cupy

Solution 1:[1]

Sources

Related Questions

Solution 1:^[1]