'Writing customised scoring function called HTER (average of false positive rate and false negative rate) in 20 runs 10 folds cross validation for CNN

I want to find mean & std of 20 runs of 10 folds cross validation for the evaluation metric HTER(Half Total Error Rate) which is average of false positive and false negative rate for CNN model. I don't know how to use the scoring parameter in cross_validate or cross_val_score. When I use make_scorer I have to go to lower version of sklearn. I did that and got this error:

TypeError: Cannot clone object '<keras.engine.sequential.Sequential object at 
0x7ff88f5f6750>' (type <class 'keras.engine.sequential.Sequential'>): it does not seem to 
be a scikit-learn estimator as it does not implement a 'get_params' methods.

Mu code;

from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_validate

# Function to create model, required for KerasClassifier
def create_model():
  # create model
    model_1 = models.Sequential()
    model_1.add(layers.Conv1D(16, 64, activation='relu', strides=1, padding="same",input_shape=(X_train_cnn.shape[1], 1)))
    model_1.add(layers.BatchNormalization(renorm=True))
    model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))

    model_1.add(layers.Conv1D(32, 3, activation='relu', strides=1, padding="same"))
    model_1.add(layers.BatchNormalization(renorm=True))
    model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))

    model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
    model_1.add(layers.BatchNormalization(renorm=True))
    model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))

    model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
    model_1.add(layers.BatchNormalization(renorm=True))
    model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))

    model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
    model_1.add(layers.BatchNormalization(renorm=True))
    model_1.add(layers.Flatten())
    model_1.add(layers.Dropout(0.5))

    model_1.add(layers.Dense(200, activation='gelu'))
    model_1.add(layers.Dropout(0.5))
    model_1.add(layers.Dense(3, activation = 'softmax'))

    # Compile model
    model_1.compile(loss = 'sparse_categorical_crossentropy', optimizer = "adam", metrics = ['accuracy'])
    return model_1

model_1 = KerasClassifier(build_fn=create_model, epochs=50, batch_size=32, verbose=0)
#Customised function has been written to find mean & std of HTER(Half Total Error Rate)
def confusion_matrix_scorer(y_test, pred_y):
  pred = model_1.predict(X_test_cnn)
  pred_y = pred.argmax(axis=-1)
  cm = metrics.confusion_matrix(y_test, pred_y) 
  y_test = y_test.reshape(-1,1)
  pred_y = pred_y.reshape(-1,1)
  TN = cm[0][0]
  FN = cm[1][0]
  TP = cm[1][1]
  FP = cm[0][1]  
  FAR = FPR = FP/(FP+TN)
  FRR = FNR = FN/(TP+FN)
  HTER=(FAR+FRR)/2
  return HTER
# evaluate using 10-fold cross validation
scoring={'confusion_matrix_scorer': make_scorer(confusion_matrix_scorer)}
kfold = RepeatedStratifiedKFold(n_splits=10,n_repeats=20, random_state=1)
results = cross_validate(model_1, X_test_cnn, y_test, scoring=scoring, cv=kfold)
print(results.mean())
print(results.std())

How to resolve this?



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source