Python Multiprocessing Help Exit On Condition

I'm breaking my teeth on multiprocessing within Python but I'm not having any luck wrapping my head around the subject. Basically I have a procedure that is time consuming to run.

Solution 1:

Maybe something like this is what you're looking for? Keep in mind I'm writing for Python 3. Your print statement above is Python 2, in which case a side note would be to use xrange instead of range.

from argparse import ArgumentParser
from random import random
from subprocess import Popen
from sys import exit
from time import sleep

defadd_something(i):

    # Sleep to simulate the long calculation
    sleep(random() * 30)
    return i + 1defrun_my_process():

    # Start up all of the processes, pass i as command line argument# since you have your function in the same file, we'll have to handle that# inside 'main' below
    processes = []
    for i inrange(100):
        processes.append(Popen(['python', 'thisfile.py', str(i)]))

    # Wait for your desired process result# Might want to add a short sleep to the loop
    done = Falsewhilenot done:
       for proc in processes:
            returncode = proc.poll()
            if returncode == 90:
                done = Truebreak# Kill any process that are still runningfor proc in processes:

        if proc.returncode isNone:

            # Might run into a race condition here,# so might want to wrap with try block
            proc.kill()

if __name__ == '__main__':

    # Look for optional i argument here
    parser = ArgumentParser()
    parser.add_argument('i', type=int, nargs='?')
    i = parser.parse_args().i

    # If there isn't an i, then run the whole thingif i isNone:
        run_my_process()

    else:
        # Otherwise, run your expensive calculation and return the result
        returncode = add_something(i)
        print(returncode)
        exit(returncode)

EDIT:

Here's a somewhat cleaner version that uses the multiprocessing module instead of subprocess:

from random import random
from multiprocessing import Process
from sys import exit
from time import sleep

defadd_something(i):

    # Sleep to simulate the long calculation
    sleep(random() * 30)

    exitcode = i + 1print(exitcode)
    exit(exitcode)

defrun_my_process():

    # Start up all of the processes
    processes = []
    for i inrange(100):
        proc = Process(target=add_something, args=[i])
        processes.append(proc)
        proc.start()

    # Wait for the desired process result
    done = Falsewhilenot done:
        for proc in processes:
            if proc.exitcode == 90:
                done = Truebreak# Kill any processes that are still runningfor proc in processes:
        if proc.is_alive():
            proc.terminate()

if __name__ == '__main__':
    run_my_process()

EDIT 2:

Here's one last version, which I think is much better than the other two:

from random import random
from multiprocessing import Pool
from time import sleep

defadd_something(i):

    # Sleep to simulate the long calculation
    sleep(random() * 30)
    return i + 1defrun_my_process():

    # Create a process pool
    pool = Pool(100)

    # Callback function that checks results and kills the pooldefcheck_result(result):
        print(result)
        if result == 90:
            pool.terminate()

    # Start up all of the processesfor i inrange(100):
        pool.apply_async(add_something, args=[i], callback=check_result)

    pool.close()
    pool.join()

if __name__ == '__main__':
    run_my_process()