TF Lite Code is not producing the same results as regular TF Code

Question

I am in the beginning stages of learning Deep-Learning. I have a project to do image classification on a Pi 4. I want to do the training on my computer and the inferencing on the Pi 4. The training is going great. The inferencing though, makes no sense. I am inferencing using a tflite model. I want to learn tflite as well since it is made for Edge devices. I am expecting an accuracy close to 95% when inferencing, but I am getting accuracy that is equal to 40 percent. Am I inferencing wrong or are there any other problems in my code? (I am using LeNet and MNIST)

import tflite_runtime.interpreter as tflite
from sklearn.metrics import accuracy_score
import numpy as np
import os
import cv2
import argparse

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--dataset", required=True,
    help="path to input dataset")
ap.add_argument("-m", "--model", required=True,
    help="path to pre-trained model")
args = vars(ap.parse_args())

image = []
classLabels = (0,1,2,3,4,5,6,7,8,9)

interpreter = tflite.Interpreter(model_path=args['model']) # construct tflite model
interpreter.allocate_tensors()

input_details = interpreter.get_input_details() # get input and output details
output_details = interpreter.get_output_details()

height = input_details[0]['shape'][1] # process images
width = input_details[0]['shape'][2]
for files in os.listdir(args["dataset"]):
    f = os.path.join(args["dataset"],files)
    img = cv2.imread(f)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img = cv2.resize(gray, (height,width))
    image.append(img)



image_array = np.array(image,dtype=np.float32)
image_array = image_array.reshape(image_array.shape[0],28,28, 1)



interpreter.resize_tensor_input(input_details[0]['index'],[10,height,width,1])
interpreter.allocate_tensors()
interpreter.set_tensor(input_details[0]['index'], image_array)


interpreter.invoke()

output_data = interpreter.get_tensor(output_details[0]['index'])

prediction_classes = np.argmax(output_data,axis=1)
ground = [7,0,2,8,9,3,4,5,1,6]
acc = accuracy_score(ground,prediction_classes)
print("{:.2f}% Accuracy\n".format(acc*100))

i = 0
for files in os.listdir(args["dataset"]):
    print("filename-{} : Pred-{}".format(files,classLabels[prediction_classes[i]]))
    i+=1