aligning face image and merge in python cv2

Question

I have a bunch of face image dataset (taken from http://vision.ucsd.edu/content/yale-face-database ) that I basically want to turn into a gif of the scramble suit from the movie scanner darkly ( http://2.bp.blogspot.com/-tRLWSOqh84Y/VSb_cF7sOoI/AAAAAAAAAWI/3XqT6d_exso/s1600/scramble%2Bsuit%2B2.gif ). So far, I am able to take the images and cut them into face "pieces" in python in bulk. The next step I am unable to do is to "align" these faces so that all the pieces form a face when they are merged or put back together. Im also unsure how to merge or put them back together. Once i have a bunch of images of randomly pieced together images, i am able to create the gif myself.

here is the code i have so far of taking the images, converting them to jpg and cutting them into necessary pieces (which was taken from here https://leslietj.github.io/2020/06/30/Automatic-Face-Crop-Using-Dlib/ ):

import sys
import dlib
from skimage import io
import numpy as np
import cv2
import matplotlib.pylab as plt
import math
from PIL import Image
import os

def arc_points(point1, point2, num_of_points):
    points = []
    center_x = (point1[0] + point2[0])/2
    center_y = (point1[1] + point2[1])/2
    radius = abs((point1[0] - point2[0])/2)
    for i in range(num_of_points):
        if i == 0:
            continue
    
        point = []
        x = center_x + radius * math.cos(math.pi + i * math.pi / num_of_points)
        y = center_y + radius * math.sin(math.pi + i * math.pi / num_of_points)
        point.append(x)
        point.append(y)
        
        points.append(point)
    
    return points


def get_landmarks(img,mode=1):
    dets = detector(img, 1)
    landmarks = np.zeros((34, 2))
    for k, d in enumerate(dets):
        shape = predictor(img, d)

        #quarter face (#1)
        if mode == 1:
            landmarks[0]= (shape.part(0).x, shape.part(0).y)
            landmarks[1] = (shape.part(1).x, shape.part(1).y)
            landmarks[2] = (shape.part(2).x, shape.part(2).y)
            landmarks[3] = (shape.part(30).x, shape.part(30).y)
            landmarks[4] = (shape.part(29).x, shape.part(29).y)
            landmarks[5] = (shape.part(28).x, shape.part(28).y)
            point1 = [shape.part(0).x, shape.part(0).y]
            point2 = [shape.part(28).x, shape.part(28).y]
            points = arc_points(point1, point2, 29)
            for i in range(len(points)):
                landmarks[33 - i] = (points[i][0], points[i][1])
        

        #half face (#2)

        if mode == 2:
            landmarks[0] = (shape.part(0).x, shape.part(0).y)
            landmarks[1] = (shape.part(1).x, shape.part(1).y)
            landmarks[2] = (shape.part(2).x, shape.part(2).y)
            landmarks[3] = (shape.part(14).x, shape.part(14).y)
            landmarks[4] = (shape.part(15).x, shape.part(15).y)
            landmarks[5] = (shape.part(16).x, shape.part(16).y)
            point1 = [shape.part(0).x, shape.part(0).y]
            point2 = [shape.part(16).x, shape.part(16).y]
            points = arc_points(point1, point2, 29)
            #print(points)
            for i in range(len(points)):
                #print(33-i)
                landmarks[33 - i] = (points[i][0], points[i][1])
        
        if mode == 3:
            #3/4 face (#3)
            for i in range(9):
                landmarks[i] = (shape.part(i).x, shape.part(i).y)
            landmarks[9] = (shape.part(31).x, shape.part(31).y)
            
            landmarks[10] = (shape.part(14).x, shape.part(14).y)
            landmarks[11] = (shape.part(15).x, shape.part(15).y)
            landmarks[12] = (shape.part(16).x, shape.part(16).y)
            point1 = [shape.part(0).x, shape.part(0).y]
            point2 = [shape.part(16).x, shape.part(16).y]
            points = arc_points(point1, point2, 22)
            for i in range(len(points)):
                landmarks[33 - i] = (points[i][0], points[i][1])
        #full face (#4)
        if mode == 4:
            for i in range(17):
                landmarks[i] = (shape.part(i).x, shape.part(i).y)
            point1 = [shape.part(0).x, shape.part(0).y]
            point2 = [shape.part(16).x, shape.part(16).y]
            points = arc_points(point1, point2, 18)
            for i in range(len(points)):
                landmarks[33 - i] = (points[i][0], points[i][1])
        

    return landmarks


def inside(X,Y,Region): 
    j=len(Region)-1
    flag=False
    for i in range(len(Region)):
        if (Region[i][1]<Y and Region[j][1]>=Y or Region[j][1]<Y and Region[i][1]>=Y):  
            if (Region[i][0] + (Y - Region[i][1]) / (Region[j][1] - Region[i][1]) * (Region[j][0] - Region[i][0]) < X):
                flag =not flag
        j=i
    return flag

count=0
files = os.listdir('yalefaces')
for filename in files:
    if filename.endswith('glasses') or filename.endswith('happy') or filename.endswith('noglasses') or filename.endswith('normal'):
        path = os.path.join('yalefaces',filename)

  
        # importing the image 
        im = Image.open(path)

      
        # converting to jpg
        rgb_im = im.convert("RGB")
      
        # exporting the image
        rgb_im.save('temp.jpg')

        count+=1


        path = 'temp.jpg'
        for im in range(1,5):
            #path = 'subject01.jpg'
            detector = dlib.get_frontal_face_detector() 
        # the .dat file can be downloaded following this link:
        # https://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2/download
            predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
            img = io.imread(path)

            region = get_landmarks(img,mode=im)
            shape = list(img.shape) 
            cropped_img = img.copy()
            for i in range(shape[0]):
                for j in range(shape[1]):
                    if not inside(j, i, region):
                        #print(img[0])
                        cropped_img[i, j] = (img[0,0][0], img[0,0][1], img[0,0][2]) # the RGB values of the background


            cropped_img = cv2.cvtColor(cropped_img, cv2.COLOR_BGR2GRAY)
            cv2.imwrite(str(count).zfill(3)+'-'+str(im).zfill(2)+'.jpg', cropped_img)

Answer 1

since this is several steps of pre-processing:

1. align faces using this script https://pyimagesearch.com/2017/05/22/face-alignment-with-opencv-and-python/
2. cut up the faces and put them together. as mentioned before, i was doing a cv2.add which is not what i want. i actually want the layers to stack up and if there is anything underneath, ignore it. cv2.add doesnt do this so i had to make my own. so if i had two layers and i want to stack them up so that the first layer on top is the one that takes priority and the bottom layer is ignored if there is anything above it.

def reduction(layer1,layer2):
    for i in range(0,layer1.shape[0]):
        for j in range(0,layer1.shape[1]):
            pixel1 = layer1.item(i, j)
            pixel2 = layer2.item(i, j)
            if layer2[i,j] != 255:
                layer1[i,j]=255
    return layer1

layer2 = reduction(layer2,layer1)

for i in range(0,layer1.shape[0]):
    for j in range(0,layer1.shape[1]):
        pixel = layer2[i,j]
        if pixel != 255:
            layer1[i,j]=layer2[i,j]

and thats it. i didnt realize images are just numpy arrays so i can just manipulate the arrays directly.