Is there any way to stop all workers in a Threadpool if any of the worker raise an Exception in python

Question

I am writing a python script to process an operation through the ThreadPoolExecutor. My requirement is to stop script execution if the exception is raised by any of the workers. I have used the exit method but it only exit/stops the particular thread, not the whole script.

Below is the piece of code:

def create_dump():
    tenants = get_all_tenants()
    with ThreadPoolExecutor(max_workers=8) as executor:
        executor.map(process_create_dump, tenants, chunksize=1)
    file = open(dump_file_path, 'a')
    json.dump(json_dump, file, indent=1)
    file.close()

def process_create_dump(tenant):
    r_components = dict()
    print("processing.....%s" % tenant)
    try:
        add_clients(r_components, tenant)
    except Exception:
        print("Unexpected exception occurred while processing")
        exit(1)

Answer 1

I found a solution using threading.Event and shutdown method of ThreadPoolExecutor. Below is the code:

event_object = threading.Event()


def create_dump():
    tenants = get_all_tenants()
    with ThreadPoolExecutor(max_workers=8) as executor:
        executor.submit(terminate_executor, executor)
        executor.map(process_create_dump, tenants, chunksize=1)
        executor.submit(set_event_on_success)
    file = open(dump_file_path, 'a')
    json.dump(json_dump, file, indent=1)
    file.close()


def process_create_dump(tenant):
    r_components = dict()
    print("processing.....%s" % tenant)
    try:
        add_clients(r_components, tenant)
    except Exception:
        event_object.set()


def terminate_executor(executor):
    event_object.wait()
    executor.shutdown(wait=False, cancel_futures=True)


def set_event_on_success():
    while not is_process_completed():
        pass
    event_object.set()