Delaunay face swap on webcam, OpenCV gives !_src.empty() error

Question

I have two webcam feeds and I want to swap faces between them using OpenCV and triangulation. So far I'm able to move sourceframe face to replace destinationframe face. However now I try to do it other way around. With current code, I get the following error:

new_source_face_canvas_area_gray = cv2.cvtColor(new_source_face_canvas_area, cv2.COLOR_BGR2GRAY)
cv2.error: OpenCV(4.5.3) /tmp/pip-install-7krwkm_h/opencv-contrib-python_1e8bcbc3da2744c690bb65b52d8197bb/opencv/modules/imgproc/src/color.cpp:182: 
error: (-215:Assertion failed) !_src.empty() in function 'cvtColor'

Is anyone able to point out what's happening there? I've tried to be logical when reversing the face swap from destionationframe to sourceframe, but I'm sure I'm doing silly mistake when trying to make everything "the opposite". Many thanks for your help!

                sourceframe = video_process.frame
                destinationframe = video_process.frame2

                height, width, channels = destinationframe.shape 
                height2, width2, channels2 = sourceframe.shape

                sourcegray = cv2.cvtColor(sourceframe, cv2.COLOR_BGR2GRAY) 
                destinationgray = cv2.cvtColor(destinationframe, cv2.COLOR_BGR2GRAY) 

                sourcemask = np.zeros_like(sourcegray)
                destinationmask = np.zeros_like(destinationgray)

                source_image_canvas = np.zeros((height2, width2, channels2), np.uint8)
                destination_image_canvas = np.zeros((height, width, channels), np.uint8)

                indexes_triangles = []

                #landmarks of first face

                sourcefaces = video_process.faces 
                destinationfaces = video_process.faces2 

                for sourceface in sourcefaces:
                    sourcelandmarks = video_process.landmarks
                    sourcelandmarks_points = []
                    for n in range(0, 68):
                        x = sourcelandmarks.part(n).x
                        y = sourcelandmarks.part(n).y
                        sourcelandmarks_points.append((x, y))

                    source_triangle_points = np.array(sourcelandmarks_points, np.int32)
                    sourceconvexhull = cv2.convexHull(source_triangle_points)
                    cv2.fillConvexPoly(sourcemask, sourceconvexhull, 255)    
                    

                    # Delaunay triangulation

                    sourcerect = cv2.boundingRect(sourceconvexhull)
                    sourcesubdiv = cv2.Subdiv2D(sourcerect)
                    sourcesubdiv.insert(sourcelandmarks_points)
                    sourcetriangles = sourcesubdiv.getTriangleList()
                    sourcetriangles = np.array(sourcetriangles, dtype=np.int32)
                    

                    for t in sourcetriangles:
                        pt1 = (t[0], t[1])
                        pt2 = (t[2], t[3])
                        pt3 = (t[4], t[5])

                        index_pt1 = np.where((source_triangle_points == pt1).all(axis=1))
                        index_pt1 = extract_index_nparray(index_pt1)

                        index_pt2 = np.where((source_triangle_points == pt2).all(axis=1))
                        index_pt2 = extract_index_nparray(index_pt2)

                        index_pt3 = np.where((source_triangle_points == pt3).all(axis=1))
                        index_pt3 = extract_index_nparray(index_pt3)
                        
                        if index_pt1 is not None and index_pt2 is not None and index_pt3 is not None:
                            source_triangle = [index_pt1, index_pt2, index_pt3]
                            indexes_triangles.append(source_triangle)

                # Face 2
                for destinationface in destinationfaces:
                    destinationlandmarks = video_process.landmarks2
                    destinationlandmarks_points = []
                    for n in range(0, 68):
                        x = destinationlandmarks.part(n).x
                        y = destinationlandmarks.part(n).y
                        destinationlandmarks_points.append((x, y))

                    destination_triangle_points = np.array(destinationlandmarks_points, np.int32)
                    destinationconvexhull = cv2.convexHull(destination_triangle_points)
                    cv2.fillConvexPoly(destinationmask, destinationconvexhull, 255)  


                # Iterating through all source delaunay triangle and superimposing source triangles in empty destination canvas after warping to same size as destination triangles' shape
                for triangle_index in indexes_triangles:

                    # Triangulation of the first face
                    tr1_pt1 = sourcelandmarks_points[triangle_index[0]]
                    tr1_pt2 = sourcelandmarks_points[triangle_index[1]]
                    tr1_pt3 = sourcelandmarks_points[triangle_index[2]]
                    source_triangle = np.array([tr1_pt1, tr1_pt2, tr1_pt3], np.int32)

                    # Source rectangle
                    source_rectangle = cv2.boundingRect(source_triangle)
                    (x, y, w, h) = source_rectangle
                    (xu, yu, wu, hu) = source_rectangle
                    cropped_source_rectangle = sourceframe[y: yu + hu, x: xu + wu]
                    cropped_source_rectangle_mask = np.zeros((hu, wu), np.uint8)

                    source_triangle_points = np.array([[tr1_pt1[0] - x, tr1_pt1[1] - y],
                                    [tr1_pt2[0] - x, tr1_pt2[1] - y],
                                    [tr1_pt3[0] - x, tr1_pt3[1] - y]], np.int32)

                    cv2.fillConvexPoly(cropped_source_rectangle_mask, source_triangle_points, 255)



                    # Triangulation of second face
                    tr2_pt1 = destinationlandmarks_points[triangle_index[0]]
                    tr2_pt2 = destinationlandmarks_points[triangle_index[1]]
                    tr2_pt3 = destinationlandmarks_points[triangle_index[2]]
                    destination_triangle = np.array([tr2_pt1, tr2_pt2, tr2_pt3], np.int32)


                    # Dest rectangle
                    destination_rectangle = cv2.boundingRect(destination_triangle)
                    (x, y, w, h) = destination_rectangle

                    cropped_destination_rectangle = sourceframe[y: y + h, x: x + w]  
                    cropped_destination_rectangle_mask = np.zeros((h, w), np.uint8)

                    destination_triangle_points = np.array([[tr2_pt1[0] - x, tr2_pt1[1] - y],
                                        [tr2_pt2[0] - x, tr2_pt2[1] - y],
                                        [tr2_pt3[0] - x, tr2_pt3[1] - y]], np.int32)

                    cv2.fillConvexPoly(cropped_destination_rectangle_mask, destination_triangle_points, 255)


                    # Warp source triangle to match shape of destination triangle and put it over destination triangle mask
                    source_triangle_points = np.float32(source_triangle_points)
                    destination_triangle_points = np.float32(destination_triangle_points)

                    matrix = cv2.getAffineTransform(source_triangle_points, destination_triangle_points)
                    matrix2 = cv2.getAffineTransform(destination_triangle_points, source_triangle_points)

                    warped_rectangle = cv2.warpAffine(cropped_source_rectangle, matrix, (w, h))
                    warped_triangle = cv2.bitwise_and(warped_rectangle, warped_rectangle, mask=cropped_destination_rectangle_mask)

                    warped_rectangle_2 = cv2.warpAffine(cropped_destination_rectangle, matrix2, (wu, hu)) 
                    warped_triangle_2 = cv2.bitwise_and(warped_rectangle_2, warped_rectangle_2, mask=cropped_source_rectangle_mask)

                
                    #  Reconstructing destination face in empty canvas of destination image
                    new_dest_face_canvas_area = destination_image_canvas[y: y + h, x: x + w] # h y etc. are from dest rect and it works
                    new_dest_face_canvas_area_gray = cv2.cvtColor(new_dest_face_canvas_area, cv2.COLOR_BGR2GRAY)

                    _, mask_created_triangle = cv2.threshold(new_dest_face_canvas_area_gray, 1, 255, cv2.THRESH_BINARY_INV)
                    warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=mask_created_triangle)

                    new_dest_face_canvas_area = cv2.add(new_dest_face_canvas_area, warped_triangle)
                    destination_image_canvas[y: y + h, x: x + w] = new_dest_face_canvas_area

                    # Reconstructing source face in empty canvas of source image

                    new_source_face_canvas_area = source_image_canvas[y: yu + hu, x: xu + wu]    # hu et are from source rect now
                    new_source_face_canvas_area_gray = cv2.cvtColor(new_source_face_canvas_area, cv2.COLOR_BGR2GRAY)
                    _, mask_created_triangle_2 = cv2.threshold(new_source_face_canvas_area_gray, 1, 255, cv2.THRESH_BINARY_INV)
                    warped_triangle_2 = cv2.bitwise_and(warped_triangle_2, warped_triangle_2, mask=mask_created_triangle_2)

                    new_source_face_canvas_area = cv2.add(new_source_face_canvas_area, warped_triangle_2)
                    source_image_canvas[y: yu + hu, x: xu + wu] = new_source_face_canvas_area