Sometimes Low Accuracy- Sometimes High Accuracy in Same Code and Dataset

Question

I'm new at CapsulNet. Using CapsNet Keras code of XifengGuo. https://github.com/XifengGuo/CapsNet-Keras/tree/tf2.2 The code is running on MNIST dataset very well, but when i trained model with my own dataset, sometimes i get accuracy %48, but sometimes %95. When it is %48, accuracy showing the same value every epoch.I also tried with different datasets. But it did the same. How can i solve it? Please help me. I convert images to grayscale. It turn MNIST format.

Thanks everybody. I have add capslayer code.

    import tensorflow as tf
    import tensorflow.keras.backend as K
    from tensorflow.keras import initializers, layers


    class Length(layers.Layer):

        def call(self, inputs, **kwargs):
            return tf.sqrt(tf.reduce_sum(tf.square(inputs), -1)+ K.epsilon())

        def compute_output_shape(self, input_shape):
            return input_shape[:-1]

        def get_config(self):
            config = super(Length, self).get_config()
            return config


     class Mask(layers.Layer):

        def call(self, inputs, **kwargs):
            if type(inputs) is list:  # true label is provided with shape = [None, n_classes], i.e. one-hot code.
                assert len(inputs) == 2
                inputs, mask = inputs
            else:  # if no true label, mask by the max length of capsules. Mainly used for prediction
        # compute lengths of capsules
                x = tf.sqrt(tf.reduce_sum(tf.square(inputs), -1))
        # generate the mask which is a one-hot code.
        # mask.shape=[None, n_classes]=[None, num_capsule]

                mask = tf.one_hot(indices=tf.argmax(x, 1), depth=x.shape[1])

    # inputs.shape=[None, num_capsule, dim_capsule]
    # mask.shape=[None, num_capsule]
    # masked.shape=[None, num_capsule * dim_capsule]
            masked = K.batch_flatten(inputs * tf.expand_dims(mask, -1))
    
            return masked

        def compute_output_shape(self, input_shape):
            if type(input_shape[0]) is tuple:  # true label provided
                return tuple([None, input_shape[0][1] * input_shape[0][2]])
            else:  # no true label provided
                return tuple([None, input_shape[1] * input_shape[2]])

        def get_config(self):
            config = super(Mask, self).get_config()
            return config


    def squash(vectors, axis=-1):

        s_squared_norm = tf.reduce_sum(tf.square(vectors), axis, keepdims=True)
        scale = s_squared_norm / (1 + s_squared_norm) / tf.sqrt(s_squared_norm + K.epsilon())
        return scale * vectors


    class CapsuleLayer(layers.Layer):

        def __init__(self, num_capsule, dim_capsule, routings=3,
             kernel_initializer='glorot_uniform',
             **kwargs):
            super(CapsuleLayer, self).__init__(**kwargs)
            self.num_capsule = num_capsule
            self.dim_capsule = dim_capsule
            self.routings = routings
            self.kernel_initializer = initializers.get(kernel_initializer)

        def build(self, input_shape):
            assert len(input_shape) >= 3, "The input Tensor should have shape=[None, input_num_capsule, input_dim_capsule]"
            self.input_num_capsule = input_shape[1]
            self.input_dim_capsule = input_shape[2]

    # Transform matrix, from each input capsule to each output capsule, there's a unique weight as in Dense layer.
            self.W = self.add_weight(shape=[self.num_capsule, self.input_num_capsule,
                                    self.dim_capsule, self.input_dim_capsule],
                             initializer=self.kernel_initializer,
                             name='W')


            self.built = True

        def call(self, inputs, training=None):
    # inputs.shape=[None, input_num_capsule, input_dim_capsule]
    # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule, 1]

            inputs_expand = tf.expand_dims(tf.expand_dims(inputs, 1), -1)

    # Replicate num_capsule dimension to prepare being multiplied by W
    # inputs_tiled.shape=[None, num_capsule, in
    # put_num_capsule, input_dim_capsule, 1]
            inputs_tiled = tf.tile(inputs_expand, [1, self.num_capsule, 1, 1, 1])

    # Compute `inputs * W` by scanning inputs_tiled on dimension 0.
    # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule]
    # x.shape=[num_capsule, input_num_capsule, input_dim_capsule, 1]
    # Regard the first two dimensions as `batch` dimension, then
    # matmul(W, x): [..., dim_capsule, input_dim_capsule] x [..., input_dim_capsule, 1] -> [..., dim_capsule, 1].
    # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule]
            inputs_hat = tf.map_fn(lambda x: tf.matmul(self.W, x), elems=inputs_tiled)
            inputs_hat = tf.squeeze(inputs_hat,4)

    # Begin: Routing algorithm ---------------------------------------------------------------------#
    # The prior for coupling coefficient, initialized as zeros.
    # b.shape = [None, self.num_capsule, 1, self.input_num_capsule].
            b = tf.zeros(shape=[tf.shape(inputs_hat)[0], self.num_capsule, 1, self.input_num_capsule])
            assert self.routings > 0, 'The routings should be > 0.'
            for i in range(self.routings):
        # c.shape=[batch_size, num_capsule, 1, input_num_capsule]
                c = tf.nn.softmax(b, axis=1)

        # c.shape = [batch_size, num_capsule, 1, input_num_capsule]
        # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
        # The first two dimensions as `batch` dimension,
        # then matmal: [..., 1, input_num_capsule] x [..., input_num_capsule, dim_capsule] -> [..., 1, dim_capsule].
        # outputs.shape=[None, num_capsule, 1, dim_capsule]
                print('c= ',c.shape)
                outputs= tf.matmul(c, inputs_hat)
                outputs=squash(outputs)
        #outputs = squash(outputs)  # [None, 10, 1, 16]
                if i < self.routings - 1:
            # outputs.shape =  [None, num_capsule, 1, dim_capsule]
            # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule]
            # The first two dimensions as `batch` dimension, then
            # matmal:[..., 1, dim_capsule] x [..., input_num_capsule, dim_capsule]^T -> [..., 1, input_num_capsule].
            # b.shape=[batch_size, num_capsule, 1, input_num_capsule]
                    b += tf.matmul(outputs, inputs_hat, transpose_b=True)
    # End: Routing algorithm -----------------------------------------------------------------------#
    
            outputs = tf.squeeze(outputs,2)
            return outputs

        def compute_output_shape(self, input_shape):
            return tuple([None, self.num_capsule, self.dim_capsule])

        def get_config(self):
            config = {
                'num_capsule': self.num_capsule,
                'dim_capsule': self.dim_capsule,
                'routings': self.routings
              }
            base_config = super(CapsuleLayer, self).get_config()
            return dict(list(base_config.items()) + list(config.items()))


    def PrimaryCap(inputs, dim_capsule, n_channels, kernel_size, strides, padding):

        output = layers.Conv2D(filters=dim_capsule*n_channels, kernel_size=kernel_size, strides=strides, padding=padding,
                       name='primarycap_conv2d')(inputs)
        outputs = layers.Reshape(target_shape=[-1, dim_capsule], name='primarycap_reshape')(output)
        outputs = layers.Lambda(squash, name='primarycap_squash')(outputs)
        return outputs