Running memory expensive python script with concurrent.futures

Question

I am running memory expensive python for different parameter combinations over and over. I am doing this on the computational cluster using slurm. (IDK if this is relevant but just wanted to mention it). My code does create the output files but all the elements of a dictionary or multiple dictionaries (depending on the version of the code) are returned empty.

I have two questions for our community:

1. Why is this happening? What am I doing wrong here?
2. Is there a better way to set up this loop and concurrency? (I have one node and <32 nodes available) and the memory required for this is north of 310GB

The main part of the code that uses concurent.futures module is set up as follows: where Crun_gridsearch is my main function and param_comb is my parameter space. ''' if name == 'main':

gc_timer = time.perf_counter()
PSPs={}; BOLDs={}; y0psps_means={}; wFIC_arrs={}; sim_FCs={}; Rs={}; absRs={}; low_PSDs={};
with concurrent.futures.ProcessPoolExecutor(max_workers=30) as executor:
    future_obj =[executor.submit(Crun_gridsearch, param_comb) for param_comb in param_combs]
    for fut in concurrent.futures.as_completed(future_obj):
        try:
            print('AS process is done')
            #PSPs={}; BOLDs={}; y0psps_means={}; wFIC_arrs={}; sim_FCs={}; Rs={}; absRs={}; low_PSDs={};
            results = fut.result()
            PSPs[results[8]]=results[0]
            BOLDs[results[8]]=results[1]
            y0psps_means[results[8]]=results[2]
            wFIC_arrs[results[8]]=results[3]
            sim_FCs[results[8]]=results[4]
            Rs[results[8]]=results[5]
            absRs[results[8]]=results[6]
            low_PSDs[results[8]]=results[7]
            print('Gc: %d R: %d' % (results[8], results[5]))
            gc.collect()
        except ValueError:
            continue

save_dict = {'PSP':PSPs, 'BOLD':BOLDs, 'y0psps_means':y0psps_means, 'wFIC_arr': wFIC_arrs,'sim_FC':sim_FCs,
    'R':Rs,'absR': absRs,'low_PSDs': low_PSDs}
out_name = OUTPUT_LOC + 'Gc_search_'+SUB+'.mat'
sio.savemat(out_name, mdict=save_dict)
print('GC search for %s finished, elapsed time: %d seconds' %
          (SUB, numpy.round(time.perf_counter()-gc_timer,2)))

'''

or in another version:

'''

if name == 'main':

gc_timer = time.perf_counter()
#PSPs={}; BOLDs={}; y0psps_means={}; wFIC_arrs={}; sim_FCs={}; Rs={}; absRs={}; low_PSDs={};
with concurrent.futures.ProcessPoolExecutor(max_workers=30) as executor:
    future_obj =[executor.submit(Crun_gridsearch, param_comb) for param_comb in param_combs]
    for fut in concurrent.futures.as_completed(future_obj):
        try:
            print('AS process is done')
            PSPs={}; BOLDs={}; y0psps_means={}; wFIC_arrs={}; sim_FCs={}; Rs={}; absRs={}; low_PSDs={}; param_str={};
            results = fut.result()
            print('T1a'+ str(results[9]))
            PSPs[results[8]]=results[0]
            BOLDs[results[8]]=results[1]
            y0psps_means[results[8]]=results[2]
            wFIC_arrs[results[8]]=results[3]
            sim_FCs[results[8]]=results[4]
            Rs[results[8]]=results[5]
            absRs[results[8]]=results[6]
            low_PSDs[results[8]]=results[7]
            save_dict = {'PSP':PSPs, 'BOLD':BOLDs, 'y0psps_means':y0psps_means, 'wFIC_arr': wFIC_arrs,'sim_FC':sim_FCs,
            'R':Rs,'absR': absRs,'low_PSDs': low_PSDs}
            print(save_dict)
            out_name = OUTPUT_LOC + 'Gc_search_'+SUB+'_gc_'+str(results[8])+'.mat'
            sio.savemat(out_name, mdict=save_dict)
            gc.collect()
        except ValueError:
            continue
print('GC search for %s finished, elapsed time: %d seconds' %
          (SUB, numpy.round(time.perf_counter()-gc_timer,2)))

'''

I would be super grateful for help and advice. Have a lovely day Jan