[PYTHON] Instant multiprocessing

(4/5 postscript) Pattern to be handed over as a pickled temporary file (9/4 postscript) Windows compatible version

Create a decorator and throw the process to the child process without rewriting the existing function.

test.py

import os
from multiprocessing import Process, Queue

#Decorator
def mp(func):
    def _f(*args, **kwargs):
        def _p(*args, **kwargs):
            al = list(args)
            q = al.pop()
            q.put(func(*al, **kwargs))
        queue = Queue()
        argsl = list(args)
        argsl.append(queue)
        proc = Process(target=_p, args=argsl, kwargs=kwargs)
        proc.start()
        proc.join()
        return queue.get()
    return _f


#Run in child process and return result
@mp
def mp_test(a, b=None):
    return (a + b, os.getpid())

result, child = mp_test(1, 2)
print("parent: {}".format(os.getpid()))
print("child: {}".format(child)
print(result)

(4/5 postscript) Pattern to be handed over as a pickled temporary file When the data size is large

test2.py

import os
import pickle
from multiprocessing import Process

#Decorator
def pickle_file(func):
    def _f(*args, **kwargs):
        def _p(*args, **kwargs):
            al = list(args)
            n = al.pop()
            res = func(*al, **kwargs)
            with open(n, "wb") as f:
                pickle.dump(res, f)
        tmp_name = "_tmp.pickle"
        argsl = list(args)
        argsl.append(tmp_name)
        proc = Process(target=_p, args=argsl, kwargs=kwargs)
        proc.start()
        proc.join()
        with open(tmp_name, "rb") as f:
            result = pickle.load(f)
        os.remove(tmp_name)
        return result
    return _f

#Run in child process and return result
@pickle_file
def mp_test(a, b=None):
    return (a + b, os.getpid())

result, child = mp_test(1, 2)
print("parent: {}".format(os.getpid()))
print("child: {}".format(child)
print(result)

(9/4 postscript) Windows compatible version In Windows, the process start method is spawn, so the function passed by target must be pickle. Nested functions cannot be pickled, so this technique cannot be used.

In the case of Windows, it is not interesting at all, but I will create a class.

test_win.py

import os
import pickle
import multiprocessing as mp


class FileBasedIPC(object):
    def __init__(self, func):
        self.func = func

    def _f(self, *args, **kwargs):
        tmp_name = "_tmp.pickle"
        argsl = list(args)
        argsl.append(tmp_name)
        proc = mp.Process(target=self._p, args=argsl, kwargs=kwargs)
        proc.start()
        proc.join()
        with open(tmp_name, "rb") as f:
            result = pickle.load(f)
        os.remove(tmp_name)
        return result

    def _p(self, *args, **kwargs):
        al = list(args)
        n = al.pop()
        res = self.func(*al, **kwargs)
        with open(n, "wb") as f:
            pickle.dump(res, f)

    def __call__(self):
        return self._f

#Run in child process and return result
def mp_test(a, b=None):
    return (a + b, os.getpid())

result, child = FileBasedIPC(mp_test)()(1, 2)
print("parent: {}".format(os.getpid()))
print("child: {}".format(child)
print(result)