Mixed Input Tensorflow Model isn't training properly

Question

Below is my code. I can get the model to train, but the accuracy is .007 at max. I don't believe I am doing this correctly as when I train the data seperately I achieve 80% accuracy. Im not entirely sure I am mapping the data correctly with the tf dataset, nor am I sure I have the multi input models structured properly. Any help is greatly appreciated. Thanks in advance.

import tensorflow as tf
import os
import pandas as pd
from keras.preprocessing import image
import numpy as np
import random

IMAGE_SIZE = [600,800]
BATCH_SIZE = 128
split_size = .95
step_divide = 25.01
model_name = "name"
strategy = tf.distribute.MirroredStrategy(devices=["/gpu:0", "/gpu:1"], cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())
import datetime
from sklearn.utils.class_weight import compute_class_weight
from tensorflow.keras.callbacks import ReduceLROnPlateau
def build_model(num_classes):
    from tensorflow_addons.optimizers import CyclicalLearningRate
    from tensorflow.keras.optimizers import Adam
    from tensorflow.keras.layers import Conv2D, MaxPool2D, Rescaling, Flatten, Dense, Dropout, Input, BatchNormalization
    from tensorflow.keras import Model
    
    def scale_fn(x):
    return 1 / (2.0 ** (x - 1))
    clrStepSize = ((split/BATCH_SIZE)/step_divide)*10
    
    cyclical_learning_rate = CyclicalLearningRate(
    initial_learning_rate=0.00000001,
    maximal_learning_rate=0.0001,
    step_size=clrStepSize,
    scale_fn=scale_fn,
    scale_mode='cycle')

    input = Input(shape=(600,800, 3), name='x')

    x = input
    x = Rescaling(scale=1./255)(x)
    x = Conv2D(32, (3, 3),activation='relu', padding='same')(x)
    x = MaxPool2D(pool_size=(2, 2))(x)
    x = Dropout(0.05)(x)
    x = Conv2D(64, (3, 3), activation='relu')(x)
    x = MaxPool2D(pool_size=(2, 2))(x)
    x = Dropout(0.05)(x)
    x = Conv2D(128, (3, 3), activation='relu')(x)
    x = MaxPool2D(pool_size=(2, 2))(x)  
    x = Dropout(0.05)(x)
    x = Conv2D(256, (3, 3), activation='relu')(x)
    x = MaxPool2D(pool_size=(2, 2))(x) 
    x = Dropout(0.05)(x)
    x = Conv2D(512, (3, 3), activation='relu')(x)
    x = MaxPool2D(pool_size=(2, 2))(x)
    x = Dropout(0.05)(x)
    x = Conv2D(1024, (3, 3), activation='relu')(x)
    x = MaxPool2D(pool_size=(2, 2))(x) 
    x = Flatten()(x)
    x = Dense(1024, activation='relu')(x)
    x = Dropout(0.5)(x)  
    x = Dense(num_classes, activation='softmax')(x)
    x  =  Model( inputs = input,  outputs = x) 

    input_b = Input(shape=(1,), name="labels_ds", dtype="int64")
    y= Dense(32, activation='relu')(input_b)
    y = Dropout(0.05)(y)
    y = Dense(context_classes, activation='linear')(y)
    y  =  Model( inputs = input_b,  outputs = y) 

    combined  =  tf.keras.layers.concatenate([ x.output,  y.output]) 
    z = Dense(2, activation='relu')(combined)
    z = Dense(1, activation='linear')(z)



    # Compile
    model = Model(inputs = [ x. input,  y. input], outputs=z, name="vidcCNN")
    model.summary()
    model.compile(optimizer=Adam(learning_rate=cyclical_learning_rate)
    , loss="sparse_categorical_crossentropy", metrics=[tf.keras.metrics.TopKCategoricalAccuracy(5),tf.keras.metrics.SparseCategoricalAccuracy()])
    return model


def parse_dont_resize(filename):
image = tf.io.read_file(filename)
image = tf.io.decode_jpeg(image, channels=3)
image = tf.image.random_brightness(image, max_delta=0.05)
image = tf.image.random_flip_left_right(image)   
image = tf.image.random_flip_up_down(image)

return image



df=pd.read_csv('D:\ohDeepThought/vidcCNN/working-model-dataset-full-800×600.csv',dtype=str,na_values='-')
df.fillna('n/a',inplace=True)

df.cns.value_counts().to_csv('D:\ohDeepThought/vidcCNN/statsdf-1.csv', encoding='utf-8')


df['allKeys'] = df.cns.astype("category").cat.codes
df['allKeys2'] = df.context.astype("category").cat.codes


df = df.sample(frac=1)


dictionary_items = dfdict.items()
sorted_items = sorted(dictionary_items)
classMap = list(sorted_items)
num_images = len(df['image'])
split = int(num_images*split_size)
split2 = int(num_images*99)
num_classes = len(df['allKeys'].unique())
context_classes = len(df['allKeys2'].unique())
labels = df['allKeys'][:split].astype(int).tolist() #os.listdir(path)
labels2 = df['allKeys2'][:split].astype(int).tolist() 
filenames = df['imagepath'][:split].tolist() #glob(path + '/*/*')
print(labels)
val_labels =  df['allKeys'][split:num_images-20].astype(int).tolist()
val_labels2 =  df['allKeys2'][split:num_images-20].astype(int).tolist()
val_filenames = df['imagepath'][split:num_images-20].tolist()

test_labels =  df['allKeys'][num_images-20:].astype(int).tolist()
test_labels2 =  df['allKeys2'][num_images-20:].astype(int).tolist()
test_filenames = df['imagepath'][num_images-20:].tolist()
print(len(val_filenames))
log_date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
filepath = 'D:\ohDeepThought/vidcCNN/'+model_name+log_date



filenames_ds = tf.data.Dataset.from_tensor_slices(filenames)
labels_ds = tf.data.Dataset.from_tensor_slices(labels2)
x = filenames_ds.map(parse_dont_resize, num_parallel_calls=tf.data.experimental.AUTOTUNE)
inps_ds = tf.data.Dataset.zip((x, labels_ds)).map(lambda x, labels_ds: {'x': x, 'labels_ds': labels_ds})
labels2_ds = tf.data.Dataset.from_tensor_slices(labels)

train_ds= tf.data.Dataset.zip((inps_ds, labels2_ds))
train_ds = train_ds.shuffle(2000).batch(BATCH_SIZE, drop_remainder=True)
train_ds = train_ds.repeat()



val_filenames_ds = tf.data.Dataset.from_tensor_slices(val_filenames)
val_labels_ds = tf.data.Dataset.from_tensor_slices(val_labels)
val_x = val_filenames_ds.map(parse_dont_resize, num_parallel_calls=tf.data.experimental.AUTOTUNE)
val_inps_ds = tf.data.Dataset.zip((val_x, val_labels_ds)).map(lambda val_x, val_labels_ds: {'x': val_x, 'labels_ds': val_labels_ds})
val_labels2_ds = tf.data.Dataset.from_tensor_slices(val_labels2)

val_ds= tf.data.Dataset.zip((val_inps_ds, val_labels2_ds))
val_ds = val_ds.batch(BATCH_SIZE, drop_remainder=True)
val_ds = val_ds.repeat()


saveEpoch = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)


log_dir = "logs/fit/"+model_name + log_date
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1000,update_freq='batch')


with strategy.scope():
model = build_model(num_classes)

model.fit(train_ds,validation_data=val_ds, batch_size=BATCH_SIZE,steps_per_epoch=int((split/BATCH_SIZE)/step_divide),validation_steps=int(((num_images-split)/BATCH_SIZE)/step_divide),#class_weight=d_class_weights, 
epochs=100,callbacks=[tensorboard_callback, saveEpoch])