使用 cupy 时出现 python 多处理错误

问题描述 投票：0回答：2

考虑在使用 cupy 进行模拟的类中使用多重处理的简化示例。 这部分

obj = FUNC(par)
obj.simulate()

用

cupy
完成，然后将数据复制到
CPU
。多重处理不会触及 
GPU
上的数据，但我仍然收到以下错误：

import tqdm
import cupy as cp
import numpy as np
from multiprocessing import Pool


class FUNC:

    def __init__(self, par) -> None:
        self.x = par['x']

    def simulate(self):

        x = self.x
        xs = []

        for t in tqdm.trange(nt):

            dx = cp.random.rand(nn, ns) * 0.001
            x += dx
            xs.append(x.get())

        self.xs = np.asarray(xs)

    def func(self, i):

        x = self.xs[:, :, i]
        return [np.mean(x)]

    def stats(self):

        with Pool(processes=2) as pool:
            data = (pool.map(self.func, range(ns)))

        return cp.asnumpy(data)


if __name__ == "__main__":

    nn = 10
    ns = 10
    nt = 2500

    par = {
        "x": cp.random.randn(nn, ns).astype('f'),
    }

    obj = FUNC(par)
    obj.simulate()
    data = obj.stats()

    print(data.shape)


Process ForkPoolWorker-1:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-2:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-3:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-4:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error
Process ForkPoolWorker-5:
Traceback (most recent call last):
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/pool.py", line 114, in worker
    task = get()
  File "/home/ziaee/anaconda3/envs/sbinmms/lib/python3.9/multiprocessing/queues.py", line 368, in get
    return _ForkingPickler.loads(res)
  File "cupy/_core/core.pyx", line 2250, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2271, in cupy._core.core.array
  File "cupy/_core/core.pyx", line 2403, in cupy._core.core._array_default
  File "cupy/_core/core.pyx", line 171, in cupy._core.core.ndarray.__init__
  File "cupy/cuda/memory.pyx", line 698, in cupy.cuda.memory.alloc
  File "cupy/cuda/memory.pyx", line 1375, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/memory.pyx", line 1395, in cupy.cuda.memory.MemoryPool.malloc
  File "cupy/cuda/device.pyx", line 48, in cupy.cuda.device.get_device_id
  File "cupy_backends/cuda/api/runtime.pyx", line 159, in cupy_backends.cuda.api.runtime.getDevice
  File "cupy_backends/cuda/api/runtime.pyx", line 132, in cupy_backends.cuda.api.runtime.check_status
cupy_backends.cuda.api.runtime.CUDARuntimeError: cudaErrorInitializationError: initialization error

我哪里做错了？

python multiprocessing cupy
2个回答
0
投票

在此处添加答案以结束此问题。在研究这个问题时没有偶然发现 Stack Overflow 线程，所以我假设这个线程将来会得到更多的浏览。

该问题与默认启动方法不适用于 CUDA 多处理有关。通过显式设置启动方法以使用 

multiprocessing.set_start_method('spawn', force=True)
生成，此问题已解决。

0
投票

我已经查看了所有答案，不幸的是，没有一个真正回答了提问者的问题。让我来结束这个问题吧。

我相信提问者一定知道“多重处理”。 set'_start_方法('spawn',force=True)'需要和CUDA多处理一起使用，他只是疑惑为什么上面的多处理操作没有涉及GPU操作，却报错。

唯一的原因就是‘自我’。您忽略了 self 中的一些复制变量，例如 self. X。当启动多个进程时，它会fork所有信息，包括这个self。 x，因此报错。

最新问题
© www.soinside.com 2019 - 2024. All rights reserved.