I have a class that I want to share in a read-only fashion with children processes in a pool, so I prepared a proxy of a class but it didn’t work. The following is a simplified example of my problem.
from multiprocessing.managers import BaseManager class TestClass: def __init__(self, a): self.a = a def b(self): print self.a class MyManager(BaseManager): pass MyManager.register('test', TestClass) if __name__ == '__main__': manager = MyManager() manager.start() t = TestClass(1) print t.a mt = manager.test(2) mt.b() mt.a
When I run this code I get:
1 2 Traceback (most recent call last): File "multiprocess_example_stackexchange.py", line 20, in <module> mt.a AttributeError: 'AutoProxy[test]' object has no attribute 'a'
It seems that I cannot access the attribute of a shared object directly via a proxy. Is the only way using a method that gets the attribute, or am I doing something wrong?
Advertisement
Answer
The Proxy
objects used by multiprocessing.BaseManager
and its sub-classes normally only expose methods from the objects they’re referring to, not attributes. Now, there is multiprocessing.Manager().Namespace
, which provides a Proxy
sub-class that does provide access to attributes, rather than methods. We can create our own Proxy
type which inherits from that, which enables access to all our attributes, as well as access to our b
function:
from multiprocessing.managers import BaseManager, NamespaceProxy class TestClass(object): def __init__(self, a): self.a = a def b(self): print self.a class MyManager(BaseManager): pass class TestProxy(NamespaceProxy): # We need to expose the same __dunder__ methods as NamespaceProxy, # in addition to the b method. _exposed_ = ('__getattribute__', '__setattr__', '__delattr__', 'b') def b(self): callmethod = object.__getattribute__(self, '_callmethod') return callmethod('b') MyManager.register('test', TestClass, TestProxy) if __name__ == '__main__': manager = MyManager() manager.start() t = TestClass(1) print t.a mt = manager.test(2) print mt.a mt.a = 5 mt.b()
Output:
1 2 5
Edit:
If you want to be able to dynamically add methods from your original class to a Proxy class, you can do something like this:
from multiprocessing.managers import BaseManager, NamespaceProxy import inspect class TestClass(object): def __init__(self, a): self.a = a def b(self): print self.a class AnotherClass(object): def __init__(self, a): self.a = a def c(self): print self.a class MyManager(BaseManager): pass class ProxyBase(NamespaceProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') class TestProxy(ProxyBase): pass class AnotherProxy(ProxyBase): pass def register_proxy(name, cls, proxy): for attr in dir(cls): if inspect.ismethod(getattr(cls, attr)) and not attr.startswith("__"): proxy._exposed_ += (attr,) setattr(proxy, attr, lambda s: object.__getattribute__(s, '_callmethod')(attr)) MyManager.register(name, cls, proxy) register_proxy('test', TestClass, TestProxy) register_proxy('another', AnotherClass, AnotherProxy) if __name__ == '__main__': manager = MyManager() manager.start() mt = manager.test(2) ma = manager.another(3) mt.b() ma.c() mt.a = 5 ma.a = 6 mt.b() ma.c()