'PicklingError: Could not pickle the task to send it to the workers. while using gridsearch for lgbm
I'm trying to find best parameter in LGBM using GridSearch and here's my approach.
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb
model = lgb.LGBMRegressor(random_state=1, objective='regression')
param_grid = {
'max_depth':[20, 30, 2],
'num_leaves': [700, 1000, 100],
'min_data_in_leaf': range(200, 500),
'n_estimators': [400, 800, 100],
'learning_rate':[0.05],
'feature_fraction':[0.4, 0.5],
'max_bin':[400, 500]}
model_l = GridSearchCV(model, param_grid, cv=5 , n_jobs=-1)
model_l.fit(df_train_robust, y_floor_train,
eval_set=[(df_test_robust, y_floor_test)],
eval_metric=['l1','l2_root','mape'])
After running for like a day I'm getting this error:
/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.
"timeout or by a memory leak.", UserWarning
exception calling callback for <Future at 0x7f2a65498ad0 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/reduction.py", line 247, in dumps
dump(obj, buf, reducers=reducers, protocol=protocol)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/reduction.py", line 240, in dump
_LokyPickler(file, reducers=reducers, protocol=protocol).dump(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/cloudpickle/cloudpickle.py", line 482, in dump
return Pickler.dump(self, obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 890, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 789, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 638, in save_reduce
save(args)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 510, in save
rv = reduce(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 835, in dispatch_one_batch
self._dispatch(tasks)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 754, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 551, in apply_async
future = self._workers.submit(SafeFunction(func))
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/reusable_executor.py", line 160, in submit
fn, *args, **kwargs)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1047, in submit
self._ensure_executor_running()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1021, in _ensure_executor_running
self._adjust_process_count()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 999, in _adjust_process_count
worker_exit_lock = self._context.BoundedSemaphore(1)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/context.py", line 220, in BoundedSemaphore
return BoundedSemaphore(value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a651b3710 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a65158090 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/queues.py", line 150, in _feed
obj_ = dumps(obj, reducers=reducers)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/_base.py", line 625, in _invoke_callbacks
callback(self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 340, in __call__
self.parallel.dispatch_next()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 769, in dispatch_next
if not self.dispatch_one_batch(self._original_iterator):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 835, in dispatch_one_batch
self._dispatch(tasks)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/parallel.py", line 754, in _dispatch
job = self._backend.apply_async(batch, callback=cb)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 551, in apply_async
future = self._workers.submit(SafeFunction(func))
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/reusable_executor.py", line 160, in submit
fn, *args, **kwargs)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1047, in submit
self._ensure_executor_running()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 1021, in _ensure_executor_running
self._adjust_process_count()
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py", line 999, in _adjust_process_count
worker_exit_lock = self._context.BoundedSemaphore(1)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/context.py", line 220, in BoundedSemaphore
return BoundedSemaphore(value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a65158310 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 156, in __init__
SemLock.__init__(self, SEMAPHORE, value, value)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/backend/synchronize.py", line 70, in __init__
unlink_now)
OSError: [Errno 28] No space left on device
exception calling callback for <Future at 0x7f2a651824d0 state=finished raised PicklingError>
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
During handling of the above exception, another exception occurred:
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 789, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 638, in save_reduce
save(args)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 510, in save
rv = reduce(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/_memmapping_reducer.py", line 339, in __call__
for dumped_filename in dump(a, filename):
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 505, in dump
NumpyPickler(f, protocol=protocol).dump(value)
File "/home/ubuntu/anaconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 292, in save
wrapper.write_array(obj, self)
File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/joblib/numpy_pickle.py", line 104, in write_array
pickler.file_handle.write(chunk.tostring('C'))
OSError: [Errno 28] No space left on device
"""
The above exception was the direct cause of the following exception:
PicklingError Traceback (most recent call last)
<ipython-input-37-ad2cda3e0f98> in <module>
1 gs_m2_rs_poly.fit(df_train_robust, y_floor_train,
2 eval_set=[(df_test_robust, y_floor_test)],
----> 3 eval_metric=['l1','l2_root','mape'])
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in fit(self, X, y, groups, **fit_params)
708 return results
709
--> 710 self._run_search(evaluate_candidates)
711
712 # For multi-metric evaluation, store the best_index_, best_params_ and
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in _run_search(self, evaluate_candidates)
1149 def _run_search(self, evaluate_candidates):
1150 """Search all candidates in param_grid"""
-> 1151 evaluate_candidates(ParameterGrid(self.param_grid))
1152
1153
~/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py in evaluate_candidates(candidate_params)
687 for parameters, (train, test)
688 in product(candidate_params,
--> 689 cv.split(X, y, groups)))
690
691 if len(out) < 1:
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in __call__(self, iterable)
1015
1016 with self._backend.retrieval_context():
-> 1017 self.retrieve()
1018 # Make sure that we get a last message telling us we are done
1019 elapsed_time = time.time() - self._start_time
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in retrieve(self)
907 try:
908 if getattr(self._backend, 'supports_timeout', False):
--> 909 self._output.extend(job.get(timeout=self.timeout))
910 else:
911 self._output.extend(job.get())
~/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py in wrap_future_result(future, timeout)
560 AsyncResults.get from multiprocessing."""
561 try:
--> 562 return future.result(timeout=timeout)
563 except LokyTimeoutError:
564 raise TimeoutError()
~/anaconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
426 raise CancelledError()
427 elif self._state == FINISHED:
--> 428 return self.__get_result()
429
430 self._condition.wait(timeout)
~/anaconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
382 def __get_result(self):
383 if self._exception:
--> 384 raise self._exception
385 else:
386 return self._result
PicklingError: Could not pickle the task to send it to the workers.
I don't know what's going wrong after running for a while, initially I might be running out of storage so I increased the size of the instance but still running into same thing. Is it because I have too many parameter to train. I have around 1.6 million records for training this with 40 variables. Or is there a better way to find hyper parameter?
Instance:
ubuntu 20.04
m5a.8Xlarge
1000GB
Solution 1:[1]
I had the same problem.
I don't know why, but setting n_jobs=1 works to me.
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|---|
| Solution 1 | Sotrosca |