'Writing customised scoring function called HTER (average of false positive rate and false negative rate) in 20 runs 10 folds cross validation for CNN
I want to find mean & std of 20 runs of 10 folds cross validation for the evaluation metric HTER(Half Total Error Rate) which is average of false positive and false negative rate for CNN model. I don't know how to use the scoring parameter in cross_validate or cross_val_score. When I use make_scorer I have to go to lower version of sklearn. I did that and got this error:
TypeError: Cannot clone object '<keras.engine.sequential.Sequential object at
0x7ff88f5f6750>' (type <class 'keras.engine.sequential.Sequential'>): it does not seem to
be a scikit-learn estimator as it does not implement a 'get_params' methods.
Mu code;
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_validate
# Function to create model, required for KerasClassifier
def create_model():
# create model
model_1 = models.Sequential()
model_1.add(layers.Conv1D(16, 64, activation='relu', strides=1, padding="same",input_shape=(X_train_cnn.shape[1], 1)))
model_1.add(layers.BatchNormalization(renorm=True))
model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))
model_1.add(layers.Conv1D(32, 3, activation='relu', strides=1, padding="same"))
model_1.add(layers.BatchNormalization(renorm=True))
model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))
model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
model_1.add(layers.BatchNormalization(renorm=True))
model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))
model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
model_1.add(layers.BatchNormalization(renorm=True))
model_1.add(layers.MaxPooling1D(2, strides=2, padding="valid"))
model_1.add(layers.Conv1D(64, 3, activation='relu', strides=1, padding="same"))
model_1.add(layers.BatchNormalization(renorm=True))
model_1.add(layers.Flatten())
model_1.add(layers.Dropout(0.5))
model_1.add(layers.Dense(200, activation='gelu'))
model_1.add(layers.Dropout(0.5))
model_1.add(layers.Dense(3, activation = 'softmax'))
# Compile model
model_1.compile(loss = 'sparse_categorical_crossentropy', optimizer = "adam", metrics = ['accuracy'])
return model_1
model_1 = KerasClassifier(build_fn=create_model, epochs=50, batch_size=32, verbose=0)
#Customised function has been written to find mean & std of HTER(Half Total Error Rate)
def confusion_matrix_scorer(y_test, pred_y):
pred = model_1.predict(X_test_cnn)
pred_y = pred.argmax(axis=-1)
cm = metrics.confusion_matrix(y_test, pred_y)
y_test = y_test.reshape(-1,1)
pred_y = pred_y.reshape(-1,1)
TN = cm[0][0]
FN = cm[1][0]
TP = cm[1][1]
FP = cm[0][1]
FAR = FPR = FP/(FP+TN)
FRR = FNR = FN/(TP+FN)
HTER=(FAR+FRR)/2
return HTER
# evaluate using 10-fold cross validation
scoring={'confusion_matrix_scorer': make_scorer(confusion_matrix_scorer)}
kfold = RepeatedStratifiedKFold(n_splits=10,n_repeats=20, random_state=1)
results = cross_validate(model_1, X_test_cnn, y_test, scoring=scoring, cv=kfold)
print(results.mean())
print(results.std())
How to resolve this?
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
