Change image color and luminosity while keeping shadows with python

Question

I have a photo representing different elements, and I have a binary mask for each of the elements. My goal is to change the color of elements at wills, while keeping shadows.

Although changing "hue" is necessary (example of re-coloring changing hue), it is not sufficient, as I may have black or white elements for which I want to match the desired color. Thus, for these elements, I should change luminosity too, but still keep shadows. And here another issue arises: strong black or white elements may have S and V or L values (in HSV or HSL color space) that are all very near to 0 or 255.

I have tested many different versions of a coloring function I've built, which you may find below (note 4 versions of the function, named "update_color{version_num}"):

def update_color(in_img, rgb: tuple, approach: int = 1, img_format: str = 'rgb', mask=None, restore_mask=None):
    # Convert Input image to hsv
    if img_format == 'rgb':
        img_hsv = cv2.cvtColor(in_img, cv2.COLOR_RGB2HSV)
    elif img_format == 'bgr':
        img_hsv = cv2.cvtColor(in_img, cv2.COLOR_BGR2HSV)
    elif img_format == 'hsv':
        img_hsv = in_img.copy()
    else:
        raise ValueError(f"'img_format' must be one in {'rgb', 'bgr', 'hsv'}. Got '{img_format}'.")

    # Set mask when not provided
    if mask is None:
        mask = np.ones_like(img_hsv[:, :, 0], dtype='bool')

    # Exit if mask is empty
    if np.sum(mask) <= 0:
        return in_img.copy()

    # Restore S and V channels in mask
    mean_s = -1
    mean_v = -1

    # Create fake color matrix
    color_matrix = np.zeros((1, 1, 3), dtype='uint8')
    color_matrix[0, 0, ::-1] = rgb

    # Extract H, S, V
    H, S, V = cv2.cvtColor(color_matrix, cv2.COLOR_BGR2HSV)[0, 0, :]

    if approach == 1:
        # Extract channels from image
        h_channel = img_hsv[:, :, 0].copy().astype('int32')
        s_channel = img_hsv[:, :, 1].copy().astype('int32')
        v_channel = img_hsv[:, :, 2].copy().astype('int32')

        # Update H
        h_channel[mask] = H

        # Update S
        s_channel_mask = s_channel[mask].copy()
        if mean_s < 0:
            mean_s = int(np.mean(s_channel[restore_mask])) if restore_mask is not None else int(np.mean(s_channel[mask]))
        scale = (255 - v_channel[mask]) * (255 - s_channel_mask) / 240**2
        s_channel[mask] += ((S - mean_s) * np.clip(scale, 0, 1)).astype('int32')
        s_channel = np.clip(s_channel, 0, 255)

        # Update V
        if mean_v < 0:
            mean_v = int(np.mean(v_channel[restore_mask])) if restore_mask is not None else int(np.mean(v_channel[mask]))
        v_channel[mask] += V - mean_v
        v_channel = np.clip(v_channel, 0, 255)

        # Create output HSV image
        out_img_hsv = np.stack((h_channel, s_channel, v_channel), axis=2).astype(np.uint8)

    elif approach == 2:
        # Extract channels from image
        h_channel = img_hsv[:, :, 0].copy().astype('int32')
        s_channel = img_hsv[:, :, 1].copy().astype('int32')
        v_channel = img_hsv[:, :, 2].copy().astype('int32')

        # Update H
        h_channel[mask] = H

        # Update S
        adjust_s = np.max(rgb) - np.min(rgb)
        if adjust_s < 10:
            S *= adjust_s / 10
        s_channel[mask] = (s_channel[mask].astype('float32') * S / 255).astype('int32')
        s_channel = np.clip(s_channel, 0, 255)

        # Update V
        if mean_v < 0:
            mean_v = int(np.mean(v_channel[restore_mask])) if restore_mask is not None else int(
                np.mean(v_channel[mask]))
        v_mask = v_channel.copy()
        v_mask_min = np.clip(v_mask - 85, v_mask // 5, 255)
        v_mask_max = np.clip(v_mask + 85, 0, 255)
        v_channel[mask] += V - mean_v
        v_channel = np.clip(v_channel, v_mask_min, v_mask_max)

        # Create output HSV image
        out_img_hsv = np.stack((h_channel, s_channel, v_channel), axis=2).astype(np.uint8)

    elif approach == 3:
        # Extract channels from image
        h_channel = img_hsv[:, :, 0].copy().astype('int32')
        s_channel = img_hsv[:, :, 1].copy().astype('int32')
        v_channel = img_hsv[:, :, 2].copy().astype('int32')

        # Update H
        h_channel[mask] = H

        # Update S
        s_channel[mask] = S

        # Update V
        if mean_v < 0:
            mean_v = int(np.mean(v_channel[restore_mask])) if restore_mask is not None else int(
                np.mean(v_channel[mask]))
        v_channel[mask] += V - mean_v
        v_channel = np.clip(v_channel, 0, 255)

        # Create output HSV image
        out_img_hsv = np.stack((h_channel, s_channel, v_channel), axis=2).astype(np.uint8)

    else:
        img_hls = cv2.cvtColor(cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR), cv2.COLOR_BGR2HLS)

        # Extract H, L, S and channels
        H, L, S = cv2.cvtColor(color_matrix, cv2.COLOR_BGR2HLS)[0, 0, :]
        h_channel = img_hls[:, :, 0].copy().astype('int32')
        l_channel = img_hls[:, :, 1].copy().astype('int32')
        s_channel = img_hls[:, :, 2].copy().astype('int32')

        h_channel[mask] = H
        l_channel[mask] += (L - np.mean(l_channel[mask])).astype('int32')
        l_channel = np.clip(l_channel, 0, 255)
        s_channel[mask] = S

        out_img_hls = np.stack((h_channel, l_channel, s_channel), axis=2).astype(np.uint8)
        out_img_hsv = cv2.cvtColor(cv2.cvtColor(out_img_hls, cv2.COLOR_HLS2BGR), cv2.COLOR_BGR2HSV)

    # Move back to selected format
    if img_format == 'rgb':
        out_img = cv2.cvtColor(out_img_hsv, cv2.COLOR_HSV2RGB)
    elif img_format == 'bgr':
        out_img = cv2.cvtColor(out_img_hsv, cv2.COLOR_HSV2BGR)
    elif img_format == 'hsv':
        out_img = out_img_hsv.copy()
    else:
        out_img = in_img.copy()
    out_img[~mask] = in_img[~mask]

    return out_img

To test a single function it is sufficient to load an image and set a color rgb = (50, 0, 0), then call update_color1(image, rgb, img_format='bgr'). However, in this case, all elements are updated at once: light elements will remain lighter and dark elements will remain darker.

Instead, I call the update function on one element at a time. Some approaches produce good results only if original element color is convenient (e.g. if element has a middle luminosity and a high saturation), while some other approaches produce a color that is more similar across elements but it is generally not satifying (e.g. shadows are completely lost on some elements).

I need to have the same shadowed color regardless of the element underneath, and keep continuity between a single element (e.g. in my use case, one element may be split in two masks, and the result of applying mask1 and mask2 must be the same of that applying mask1+mask2, so for instance taking the np.mean of a mask may not be enough since the two masks may have different mean values).

---

Here a test code and images.

Original image

Results (note that there is an overlap over two masks, but it is not the main issue here)

Test code (masks are not perfect, but they are much easier to share)

# Load image
image = cv2.imread(r".\colored-coins.jpg")
image_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

# Extract mask
colors = [(180, 122, 100), (12, 28, 245), (255, 110, 159), (25, 124, 181),
          (125, 132, 0), (116, 31, 2), (110, 110, 80), (8, 56, 30)]
h_values = [colorsys.rgb_to_hsv(*cc)[0]*180 for cc in colors]
masks = [(image_hsv[:,:,0] < h + 20) & (image_hsv[:,:,0] > h - 20) & (image_hsv[:,:,2] > 10) for h in h_values]
masks[0][:,320:] = False
masks[1][260:, :] = False
masks[1][:, 620:] = False
masks[2][350:, :] = False
masks[2][:, :-430] = False
masks[3][:155, :] = False
masks[3][520:, :] = False
masks[3][:, :445] = False
masks[4][:300, :] = False
masks[4][-75:, :] = False
masks[4][:, :-350] = False
masks[5][:-180, :] = False
masks[6][:260, :] = False
masks[6][-170:, :] = False
masks[6][:, :320] = False
masks[6][:, -350:] = False
masks[7][:350, :] = False
masks[7][:, 330:] = False

# Update color of image
new_image1, new_image2, new_image3, new_image4 = image.copy(), image.copy(), image.copy(), image.copy()
for mask in masks:
    new_image1 = update_color(new_image1, (50, 0, 0), approach=1, img_format='bgr', mask=mask)
    new_image2 = update_color(new_image2, (50, 0, 0), approach=2, img_format='bgr', mask=mask)
    new_image3 = update_color(new_image3, (50, 0, 0), approach=3, img_format='bgr', mask=mask)
    new_image4 = update_color(new_image4, (50, 0, 0), approach=4, img_format='bgr', mask=mask)

cv2.imshow(f"1", new_image1)
cv2.imshow(f"2", new_image2)
cv2.imshow(f"3", new_image3)
cv2.imshow(f"4", new_image4)
cv2.waitKey()
cv2.destroyAllWindows()