Skip to content

Instantly share code, notes, and snippets.

@bnyeggen

bnyeggen/multiprocess_with_instance_methods.py

Created July 16, 2011 14:17
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save bnyeggen/1086393 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save bnyeggen/1086393 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. bnyeggen created this gist Jul 16, 2011.
    47 changes: 47 additions & 0 deletions multiprocess_with_instance_methods.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,47 @@
    from multiprocessing import Pool
    from functools import partial

    def _pickle_method(method):
    func_name = method.im_func.__name__
    obj = method.im_self
    cls = method.im_class
    if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names
    cls_name = cls.__name__.lstrip('_')
    func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)

    def _unpickle_method(func_name, obj, cls):
    for cls in cls.__mro__:
    try:
    func = cls.__dict__[func_name]
    except KeyError:
    pass
    else:
    break
    return func.__get__(obj, cls)

    import copy_reg
    import types
    copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)

    class someClass(object):
    def __init__(self):
    pass

    def f(self, x=None):
    #can put something expensive here to verify CPU utilization
    if x is None: return 99
    return x*x

    def go(self):
    pool = Pool()
    print pool.map(self.f, range(10))

    if __name__=='__main__':
    sc = someClass()
    sc.go()
    x=[someClass(),someClass(),someClass()]
    p=Pool()
    filled_f=partial(someClass.f,x=9)
    print p.map(filled_f,x)
    print p.map(someClass.f,x)