使用 forkserver 时无法传递 multiprocessing.Manager 进行处理：无法 pickle 'weakref' 对象

Question

示例代码：

import multiprocessing

def main():
    multiprocessing.set_start_method('forkserver')
    with multiprocessing.Manager() as manager:
        proc = multiprocessing.Process(target=fun, args=('p1', manager))
        proc.start()
        proc.join()

def fun(name, manager):
    print('fun:', name, manager)

if __name__ == '__main__':
    main()

失败并出现错误：

$ /opt/python3.12.6/bin/python3 ./demo_mp_manager_forkserver.py
Traceback (most recent call last):
  File "/home/.../demo_mp_manager_forkserver.py", line 14, in <module>
    main()
  File "/home/.../demo_mp_manager_forkserver.py", line 7, in main
    proc.start()
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
                  ^^^^^^^^^^^^^^^^^
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/context.py", line 301, in _Popen
    return Popen(process_obj)
           ^^^^^^^^^^^^^^^^^^
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/popen_forkserver.py", line 35, in __init__
    super().__init__(process_obj)
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/popen_forkserver.py", line 47, in _launch
    reduction.dump(process_obj, buf)
  File "/opt/python3.12.6/lib/python3.12/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref.ReferenceType' object

Answer 1

您应该使用

manager

创建共享对象并将 那些对象传递给子进程，而不是整个管理器。

import multiprocessing

def main():
    multiprocessing.set_start_method('forkserver')
    with multiprocessing.Manager() as manager:
        shared_dict = manager.dict()
        shared_dict['foo'] = 'bar'
        proc = multiprocessing.Process(target=fun, args=('p1', shared_dict))
        proc.start()
        proc.join()

def fun(name, shared_dict):
    print('fun:', name, shared_dict.items())

if __name__ == '__main__':
    main()

输出：

$ /opt/python3.12.6/bin/python3 ./demo_mp_manager_forkserver.py
fun: p1 [('foo', 'bar')]