ValueError: `labels.shape` must equal `logits.shape` except for the last dimension. Received: labels.shape=(524288,) and logits.shape=(1024, 2560)

Question

I am relatively new in Deep learning, I am trying to train U-NET using LIDC-IDRI dataset, I am having dimension mismatch error every time I try to train the model. Imagetrain shape is (6324, 1, 512, 512).

After running the code I get this error ValueError: `labels.shape` must equal `logits.shape` except for the last dimension. Received: labels.shape=(524288,) and logits.shape=(1024, 2560)

All imports:

from glob import glob
import matplotlib.pyplot as plt
import os  
from keras.models import Sequential,load_model,Model
from keras.layers import Dense, Dropout, Activation, Flatten, ReLU, concatenate
from keras.layers import Conv2D, MaxPooling2D, SpatialDropout2D, Conv2DTranspose
from keras.layers import Input, merge, UpSampling2D, BatchNormalization, Cropping2D    
from tensorflow.keras.optimizers import Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras import backend as K
import pandas as pd
from keras.callbacks import ModelCheckpoint
import h5py
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc
from keras.losses import binary_crossentropy, sparse_categorical_crossentropy

K.set_image_data_format('channels_first')

Datas are loaded as .npy file:

noduleimages=np.load(datafolder+"/noduleimages1.npy")
nodulemasks=np.load(datafolder+"/nodulemasks1.npy")
noduleimages=noduleimages.reshape(noduleimages.shape[0],1,512,512)
nodulemasks=nodulemasks.reshape(nodulemasks.shape[0],1,512,512)
imagestrain, imagestest, maskstrain, maskstest = train_test_split(noduleimages,nodulemasks,test_size=.20)

How can I solve this problem?

---

def convolution_operation(entered_input, filters=32):
    # Taking first input and implementing the first conv block
    conv1 = Conv2D(filters, kernel_size = (3,3), padding = "same")(entered_input)
    batch_norm1 = BatchNormalization()(conv1)
    act1 = ReLU()(batch_norm1)
    
    # Taking first input and implementing the second conv block
    conv2 = Conv2D(filters, kernel_size = (3,3), padding = "same")(act1)
    batch_norm2 = BatchNormalization()(conv2)
    act2 = ReLU()(batch_norm2)
    
    return act2

def encoder(entered_input, filters=32):
    # Collect the start and end of each sub-block for normal pass and skip connections
    enc1 = convolution_operation(entered_input, filters)
    MaxPool1 = MaxPooling2D(strides = (2,2), padding='same')(enc1)
    return enc1, MaxPool1

def decoder(entered_input, skip, filters=32):
    # Upsampling and concatenating the essential features
    Upsample = Conv2DTranspose(filters, (2, 2), strides=2, padding="same")(entered_input)
    Connect_Skip = concatenate([Upsample, skip])  
    out = convolution_operation(Connect_Skip, filters)
    return out

def U_Net(Image_Size):
    # Take the image size and shape
    input1 = Input(Image_Size)
    
    # Construct the encoder blocks
    skip1, encoder_1 = encoder(input1, 32)
    skip2, encoder_2 = encoder(encoder_1, 32*2)
    skip3, encoder_3 = encoder(encoder_2, 32*4)
    skip4, encoder_4 = encoder(encoder_3, 32*8)
    
    # Preparing the next block
    conv_block = convolution_operation(encoder_4, 32*16)
    
    # Construct the decoder blocks
    decoder_1 = decoder(conv_block, skip4, 32*8)
    decoder_2 = decoder(decoder_1, skip3, 32*4)
    decoder_3 = decoder(decoder_2, skip2, 32*2)
    decoder_4 = decoder(decoder_3, skip1, 32)
    
    out = Conv2D(1, 1, padding="same", activation="sigmoid")(decoder_4)

    model = Model(input1, out)
    
    model.compile(optimizer=Adam(lr=1e-5), loss= sparse_categorical_crossentropy, metrics=[auc])
    return model

model = U_Net((1,512, 512))
model.summary()

filepath=weightsfolder+"/unet-weights-improvement.hdf5"

# model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True)

history=model.fit(imagestrain, maskstrain, batch_size=2, epochs=10, verbose=1, shuffle=True,
              callbacks=[checkpoint],validation_data=(imagestest,maskstest))

Error message:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
f:\LungNoduleDetectionClassification-master\TrainUNET.ipynb Cell 5' in <module>
     96 # model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])
     97 checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True)
---> 99 history=model.fit(imagestrain, maskstrain, batch_size=2, epochs=10, verbose=1, shuffle=True,
    100               callbacks=[checkpoint],validation_data=(imagestest,maskstest))

File ~\AppData\Roaming\Python\Python39\site-packages\keras\utils\traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     65 except Exception as e:  # pylint: disable=broad-except
     66   filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67   raise e.with_traceback(filtered_tb) from None
     68 finally:
     69   del filtered_tb

File ~\AppData\Roaming\Python\Python39\site-packages\tensorflow\python\framework\func_graph.py:1147, in func_graph_from_py_func.<locals>.autograph_handler(*args, **kwargs)
   1145 except Exception as e:  # pylint:disable=broad-except
   1146   if hasattr(e, "ag_error_metadata"):
-> 1147     raise e.ag_error_metadata.to_exception(e)
   1148   else:
   1149     raise

ValueError: in user code:

    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\training.py", line 1021, in train_function  *
        return step_function(self, iterator)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\training.py", line 1010, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\training.py", line 1000, in run_step  **
        outputs = model.train_step(data)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\training.py", line 860, in train_step
        loss = self.compute_loss(x, y, y_pred, sample_weight)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\training.py", line 918, in compute_loss
        return self.compiled_loss(
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\engine\compile_utils.py", line 201, in __call__
        loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\losses.py", line 141, in __call__
        losses = call_fn(y_true, y_pred)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\losses.py", line 245, in call  **
        return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\losses.py", line 1862, in sparse_categorical_crossentropy
        return backend.sparse_categorical_crossentropy(
    File "C:\Users\User\AppData\Roaming\Python\Python39\site-packages\keras\backend.py", line 5202, in sparse_categorical_crossentropy
        res = tf.nn.sparse_softmax_cross_entropy_with_logits(

    ValueError: `labels.shape` must equal `logits.shape` except for the last dimension. Received: labels.shape=(524288,) and logits.shape=(1024, 2560)