custom scoring function got an error: "scoring must return a number, got tf.Tensor..."

Question

I want to create a custom pinball loss function for a simple multiple layer perceptron Keras regressor. To tune the hyperparameters of the regressor, I use OptunaSearchCV (which works like scikit-learn RandomizedSearchCV).

I think the culprit of the error is the custom scoring function I made. Its function is, for each instance (row of the input data), to calculate the pinball losses of the prediction output of length 3 with corresponding quantile targets: 0.05, 0.5, and 0.95.

I have a toy example to replicate the error message.

import math
import numpy as np
import pandas as pd
from severe_weather.data_prepare.config import SalesConfig, ResultsConfig
import optuna
from sklearn.metrics import make_scorer
from optuna.distributions import IntUniformDistribution, UniformDistribution
from optuna.pruners import MedianPruner
from optuna.integration import OptunaSearchCV
from optuna.samplers import RandomSampler
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow import keras
from scikeras.wrappers import KerasRegressor
from sklearn.model_selection import train_test_split

# dummy data
df = pd.DataFrame(np.random.rand(100, 10))
train, test = train_test_split(df, test_size=0.1, random_state = 1)
x_train, y_train = train.iloc[:, :-1], train.iloc[:, -1].values.reshape(-1,1) 
x_test, y_test = test.iloc[:, :-1], test.iloc[:, -1]

# custom loss function
def pinball_loss(alpha = [0.05, 0.5, 0.95]):
    def loss(y_true, y_pred):
        err = y_true - y_pred
        par = tf.constant(np.array(alpha), dtype=tf.float32)
        return K.mean(K.maximum(par * err, (par - 1) * err), axis=-1)
    return loss

# the MLP model
def get_model(n_neurons, dropout):
    model = keras.models.Sequential()
    model.add(keras.layers.InputLayer(input_shape=[9,]))
    model.add(keras.layers.Normalization(axis=-1))
    model.add(keras.layers.Dropout(dropout))
    model.add(keras.layers.Dense(n_neurons, activation='relu'))
    model.add(keras.layers.Dense(3))
    return model

alpha = [0.05, 0.5, 0.95]

reg = KerasRegressor(get_model, n_neurons=2, dropout=0.0, loss=pinball_loss(alpha=alpha))

study = optuna.create_study(
    direction='maximize',
    sampler=RandomSampler(seed=42),
    pruner=MedianPruner())

default_trials = [study.enqueue_trial(dict(n_neurons=s, dropout=t))
                  for s in [2] for t in [0.0]]
n_trials = 1

# custom scoring function
neg_mean_pinball_loss_scorer = make_scorer(
    pinball_loss(alpha=alpha),
    greater_is_better=False)

param_distributions = {
    "n_neurons": IntUniformDistribution(2, 10),
    "dropout": UniformDistribution(0, 0.5)}

tuned_ensemble = OptunaSearchCV(
        reg, param_distributions,
        study=study, n_trials=2, n_jobs=1,
        scoring=neg_mean_pinball_loss_scorer)    

tuned_ensemble.fit(x_train, np.tile(y_train, 3), batch_size=20, epochs=5, verbose=1, validation_split=0.3)    

Here is the error message:

[W 2022-02-03 12:51:30,963] Trial 0 failed because of the following error: ValueError("scoring must return a number, got tf.Tensor(\n[-0.20005344 -0.15885974 -0.382643   -0.4041188  -0.24968709 -0.27153793\n -0.46907195 -0.09022937 -0.21099593 -0.46771994 -0.21804126 -0.35117796\n -0.24562387 -0.3742094  -0.3017985  -0.14807063 -0.2012295  -0.12404507], shape=(18,), dtype=float32) (<class 'tensorflow.python.framework.ops.EagerTensor'>) instead. (scorer=make_scorer(loss, greater_is_better=False))")
Traceback (most recent call last):
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/_optimize.py", line 213, in _run_trial
    value_or_values = func(trial)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/integration/sklearn.py", line 229, in __call__
    scores = cross_validate(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 267, in cross_validate
    results = parallel(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 1043, in __call__
    if self.dispatch_one_batch(iterator):
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 861, in dispatch_one_batch
    self._dispatch(tasks)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 779, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 208, in apply_async
    result = ImmediateResult(func)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 572, in __init__
    self.results = batch()
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 262, in __call__
    return [func(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 262, in <listcomp>
    return [func(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/utils/fixes.py", line 211, in __call__
    return self.function(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 703, in _fit_and_score
    test_scores = _score(estimator, X_test, y_test, scorer, error_score)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 794, in _score
    raise ValueError(error_msg % (scores, type(scores), scorer))
ValueError: scoring must return a number, got tf.Tensor(
[-0.20005344 -0.15885974 -0.382643   -0.4041188  -0.24968709 -0.27153793
 -0.46907195 -0.09022937 -0.21099593 -0.46771994 -0.21804126 -0.35117796
 -0.24562387 -0.3742094  -0.3017985  -0.14807063 -0.2012295  -0.12404507], shape=(18,), dtype=float32) (<class 'tensorflow.python.framework.ops.EagerTensor'>) instead. (scorer=make_scorer(loss, greater_is_better=False))
Traceback (most recent call last):
  File "/lowes/daci/severe-weather/severe_weather/hurricanes/Phase_I/modeling/lp_test.py", line 68, in <module>
    tuned_ensemble.fit(x_train, np.tile(y_train, 3), batch_size=25, epochs=5, verbose=1, validation_split=0.3)    
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/integration/sklearn.py", line 875, in fit
    self.study_.optimize(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/study.py", line 400, in optimize
    _optimize(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/_optimize.py", line 66, in _optimize
    _optimize_sequential(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential
    trial = _run_trial(study, func, catch)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/_optimize.py", line 264, in _run_trial
    raise func_err
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/study/_optimize.py", line 213, in _run_trial
    value_or_values = func(trial)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/optuna/integration/sklearn.py", line 229, in __call__
    scores = cross_validate(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 267, in cross_validate
    results = parallel(
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 1043, in __call__
    if self.dispatch_one_batch(iterator):
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 861, in dispatch_one_batch
    self._dispatch(tasks)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 779, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 208, in apply_async
    result = ImmediateResult(func)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/_parallel_backends.py", line 572, in __init__
    self.results = batch()
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 262, in __call__
    return [func(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/joblib/parallel.py", line 262, in <listcomp>
    return [func(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/utils/fixes.py", line 211, in __call__
    return self.function(*args, **kwargs)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 703, in _fit_and_score
    test_scores = _score(estimator, X_test, y_test, scorer, error_score)
  File "/home/hdpdib/.local/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 794, in _score
    raise ValueError(error_msg % (scores, type(scores), scorer))
ValueError: scoring must return a number, got tf.Tensor(
[-0.20005344 -0.15885974 -0.382643   -0.4041188  -0.24968709 -0.27153793
 -0.46907195 -0.09022937 -0.21099593 -0.46771994 -0.21804126 -0.35117796
 -0.24562387 -0.3742094  -0.3017985  -0.14807063 -0.2012295  -0.12404507], shape=(18,), dtype=float32) (<class 'tensorflow.python.framework.ops.EagerTensor'>) instead. (scorer=make_scorer(loss, greater_is_better=False))