My training loss does not decrease for hybrid cnn lstm but validation loss decreases

Question

I am trying to learn the spectrogram for two class classification in deep learning. My spectrogram shape is 113 x4225x 512 where 113 samples 4225 time samples and 113 frequency bins. By using keras libraries I have implemented the CNNLSTM network from the paper: https://ieeexplore.ieee.org/document/8978926

My training and validation loss does not decreases significantly. By the way I have 56 samples for each class is it too low? How should augment spectrograms?

import mat73
mat = mat73.loadmat('G:\data\spectrogramsfastandslow.mat')

X = mat['singlespectrogram'] 
Y= mat['label'] 

X = X.reshape(X.shape[2],X.shape[1],X.shape[0])# 113 x4225x 512


#%%
random.seed(1)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split( X, Y, test_size=0.11, random_state=1)
X_train, X_val, y_train, y_val = train_test_split( X_train, y_train, test_size=0.1, random_state=1)

#%%
import os
os.environ["CUDA_DEVİCE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] ="1"
from keras.models import Sequential
from keras.layers import Conv1D
from keras.layers import MaxPooling1D
from keras.layers import LSTM
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import TimeDistributed
from keras.callbacks import ModelCheckpoint, EarlyStopping

model = Sequential()

model.add(Conv1D(filters=4, kernel_size=5, padding='same', activation='relu',input_shape=(4225,512)))
model.add(MaxPooling1D(pool_size=4))
model.add(Conv1D(filters=4, kernel_size=5, padding='same', activation='relu',input_shape=(4225,512)))
model.add(LSTM(32))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss= 'binary_crossentropy' , optimizer= 'adam' , metrics=[ 'acc' ])

# simple early stopping
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=20)
mc = ModelCheckpoint('best_model_cnn_lstm.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)
print(model.summary())
history= model.fit(X_train, y_train,validation_data=(X_val, y_val),batch_size=32,epochs=100,callbacks =[es,mc],verbose=1)

Result: My training loss comes from 0.6934 to 0.6930, and validation loss comes from 0.69 to 0.68