how to build the CNN for the below model in python

Question

I have train a model to detect the colour of gemstones and have used cnn.My source code is available at how can I improve the accuracy rate of the below trained model using CNN.

To achieve the same goal I am trying to train another model using hsv color model and cnn.For that I have initialise the training process as below.

import os
import numpy as np
import cv2

# map colour names to HSV ranges
color_list = [
    ['red', [0, 160, 70], [10, 250, 250]],
    ['pink', [0, 50, 70], [10, 160, 250]],
    ['yellow', [15, 50, 70], [30, 250, 250]],
    ['green', [40, 50, 70], [70, 250, 250]],
    ['cyan', [80, 50, 70], [90, 250, 250]],
    ['blue', [100, 50, 70], [130, 250, 250]],
    ['purple', [140, 50, 70], [160, 250, 250]],
    ['red', [170, 160, 70], [180, 250, 250]],
    ['pink', [170, 50, 70], [180, 160, 250]]
]


def detect_main_color(hsv_image, colors):
    color_found = 'undefined'
    max_count = 0

    for color_name, lower_val, upper_val in colors:
        # threshold the HSV image - any matching color will show up as white
        mask = cv2.inRange(hsv_image, np.array(lower_val), np.array(upper_val))

        # count white pixels on mask
        count = np.sum(mask)
        if count > max_count:
            color_found = color_name
            max_count = count

    return color_found


for root, dirs, files in os.walk('C:/Users/User/Desktop/Research Project/images/'):
    f = os.path.basename(root)

    for file in files:
        img = cv2.imread(os.path.join(root, file))
        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        print(f"{file}: {detect_main_color(hsv, color_list)}")

output from above code segment is as follows:

sapphire blue_18.jpg: blue
sapphire pink_18.jpg: pink
sapphire purple_28.jpg: purple
sapphire yellow_9.jpg: yellow
....

I have started building the cnn as below.But Iam stuck in between.How can I proceed from this onwards?

import tensorflow as tf
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D, AveragePooling2D

filters = 32      # the dimensionality of the output space
kernel_size = 3   # length of the 2D convolution window
max_pool = 2 

EPOCHS = 30                                
batch_size = 32                             
iter_per_epoch =3  
val_per_epoch = 2 

img_w, img_h = 220, 220    # width and height of image

model = Sequential()

# first layer
model.add(Conv2D(batch_size, (kernel_size, kernel_size), activation='relu', padding='same', input_shape=(img_w, img_h, 3))) # 32
model.add(MaxPooling2D((max_pool, max_pool))) #reduce the spatial size of incoming features

# second layer
model.add(Conv2D(2*batch_size, (kernel_size, kernel_size), activation='relu', padding='same')) # 64
model.add(MaxPooling2D((max_pool, max_pool))) 

# third layer
model.add(Conv2D(4*batch_size, (kernel_size, kernel_size), activation='relu', padding='same')) # 128
model.add(MaxPooling2D((max_pool, max_pool))) 

# fourth layer
model.add(Conv2D(4*batch_size, (kernel_size, kernel_size), activation='relu', padding='same')) # 128
model.add(AveragePooling2D(pool_size= (2, 2), strides= (2, 2))) 

# fifth layer
model.add(Conv2D(4*batch_size, (kernel_size, kernel_size), activation='relu', padding='same')) # 128
model.add(MaxPooling2D((max_pool, max_pool))) 

model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(16*batch_size, activation='relu'))                                             # 512
model.add(Dense(5, activation='softmax'))
model.summary()

model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])

from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(              # this is the augmentation configuration used for training
        rotation_range=25,
        zoom_range=0.1,
        width_shift_range=0.1,
        height_shift_range=0.1,
        shear_range=0.2,
        horizontal_flip=True
        )

val_datagen = ImageDataGenerator()       

Any help is appreciated.

Thankyou.