sgm/data/mask.py

# Copyright (c) OpenMMLab. All rights reserved.

"""
Functions taken from https://github.com/DanBigioi/DiffusionVideoEditing


"""

import cv2
import numpy as np
import torch

" Countour from 2:15 not good for head poses "


def face_mask(img_shape, landmark_list, dtype="uint8"):
    height, width = img_shape[:2]
    mask = np.ones((height, width, 1), dtype=dtype)
    cv2.drawContours(
        mask, np.int32([landmark_list[2:15]]), -1, color=(0), thickness=cv2.FILLED
    )

    return mask


def face_mask_jaw_box(img_shape, landmark_list, dtype="uint8", kernel_size=10):
    nose = 33
    jaw = 8

    height, width = img_shape[:2]
    mask = np.ones((height, width, 1), dtype=dtype)
    combined_landmarks = np.concatenate((landmark_list[2:15], [landmark_list[33]]))

    # Draw the combined contour on the mask
    cv2.drawContours(
        mask, [np.int32(combined_landmarks)], -1, color=(0), thickness=cv2.FILLED
    )

    inverted_mask = 1 - mask
    kernel = np.ones((kernel_size, kernel_size), np.uint8)
    mask = cv2.dilate(inverted_mask, kernel, iterations=1)
    mask = np.expand_dims(
        mask, axis=-1
    )  # Add a singleton dimension to match the number of channels
    mask = 1 - mask

    cut_h = landmark_list[nose][1]

    far_left = int(np.argmin(landmark_list[:, 0]))
    far_right = int(np.argmax(landmark_list[:, 0]))
    left_up_point = np.int32([landmark_list[far_left][0], cut_h])  # 2
    right_up_point = np.int32([landmark_list[far_right][0], cut_h])  # 15
    height_landmarks = min(landmark_list[jaw, 1] + 20, height)
    left_down_point = np.int32([landmark_list[far_left][0], height_landmarks])
    right_down_point = np.int32([landmark_list[far_right][0], height_landmarks])

    # print(cut_h, cut_h + 10, height_landmarks)

    mask_box = [left_up_point, left_down_point, right_down_point, right_up_point]

    return mask, mask_box


" Stretch the tight face mask - Countour from 2:15 but dilate, not good for extreme head poses "


def face_mask_stretch(img_shape, landmark_list, dtype="uint8", kernel_size=10):
    height, width = img_shape[:2]
    mask = np.ones((height, width, 1), dtype=dtype)
    combined_landmarks = np.concatenate((landmark_list[2:15], [landmark_list[33]]))

    # Draw the combined contour on the mask
    cv2.drawContours(
        mask, [np.int32(combined_landmarks)], -1, color=(0), thickness=cv2.FILLED
    )

    # cv2.drawContours(mask, np.int32([landmark_list[2:15]]), -1, color=(0), thickness=cv2.FILLED)
    inverted_mask = 1 - mask

    kernel = np.ones((kernel_size, kernel_size), np.uint8)
    mask = cv2.dilate(inverted_mask, kernel, iterations=1)
    mask = np.expand_dims(
        mask, axis=-1
    )  # Add a singleton dimension to match the number of channels
    mask = 1 - mask

    return mask


" Small box around mouth - Use far left, far right points for extreme head poses, cut between nose and upper mouth point"


def face_mask_box_pose(img_shape, landmark_list, dtype="uint8"):
    """
    When the head pose is different than frontal then the normal cropping with landmarks does not work correctly.
    Crop using as height the middle nose point
    Take the left/right corners using the far_left and far_right landmarks
    TODO: Maybe it is better to add some more pixels to have a bigger mask, especially on large head poses
    """

    height, width = img_shape[:2]

    nose = 33
    upper_lip = 51
    jaw = 8

    nose_point_h = landmark_list[nose, 1]
    upper_lip_point = landmark_list[upper_lip, 1]
    cut_h = (upper_lip_point - nose_point_h) / 2 + nose_point_h

    # cut_h = landmark_list[nose][1]

    mask = np.ones((height, width, 1), dtype=dtype)

    far_left = int(np.argmin(landmark_list[:, 0]))
    far_right = int(np.argmax(landmark_list[:, 0]))

    left_up_point = np.int32([landmark_list[far_left][0], cut_h])  # 2
    right_up_point = np.int32([landmark_list[far_right][0], cut_h])  # 15

    height_landmarks = min(landmark_list[jaw, 1] + 20, height)
    left_down_point = np.int32([landmark_list[far_left][0], height_landmarks])
    right_down_point = np.int32([landmark_list[far_right][0], height_landmarks])

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    left_up_point,
                    left_down_point,
                    right_up_point,
                    right_down_point,
                    left_up_point,
                    right_up_point,
                    left_down_point,
                    right_down_point,
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    return mask


" Small box around mouth - Use far left, far right points for extreme head poses, cut from nose"


def face_mask_box_pose_nose(
    img_shape,
    landmark_list,
    dtype="uint8",
    get_box=False,
    pixels_above_nose=None,
    pixels_under_jaw=None,
):
    height, width = img_shape[:2]

    nose = 33
    jaw = 8

    cut_h = landmark_list[nose][1]
    if pixels_above_nose is not None:
        # this is only for inference to take a bigger mask and blend it back to the original frame
        cut_h = cut_h - pixels_above_nose

    mask = np.ones((height, width, 1), dtype=dtype)

    far_left = int(np.argmin(landmark_list[:, 0]))
    far_right = int(np.argmax(landmark_list[:, 0]))

    left_up_point = np.int32([landmark_list[far_left][0], cut_h])  # 2
    right_up_point = np.int32([landmark_list[far_right][0], cut_h])  # 15

    height_landmarks = min(landmark_list[jaw, 1] + 20, height)
    if pixels_under_jaw is not None:
        height_landmarks = min(landmark_list[jaw, 1] + pixels_under_jaw, height)
    left_down_point = np.int32([landmark_list[far_left][0], height_landmarks])
    right_down_point = np.int32([landmark_list[far_right][0], height_landmarks])

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    left_up_point,
                    left_down_point,
                    right_up_point,
                    right_down_point,
                    left_up_point,
                    right_up_point,
                    left_down_point,
                    right_down_point,
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    if get_box:
        mask_box = [left_up_point, left_down_point, right_down_point, right_up_point]
        return mask, mask_box
    else:
        return mask


def face_mask_box_pose_big(
    img_shape, landmark_list, dtype="uint8", cut_h=None, far_left=None, far_right=None
):
    height, width = img_shape[:2]
    mask = np.ones((height, width, 1), dtype=dtype)
    nose = 33
    nose_point_h = landmark_list[nose, 1]
    if cut_h is None:
        cut_h = nose_point_h

    if far_right is None and far_left is None:
        far_left = int(np.argmin(landmark_list[:, 0]))
        far_right = int(np.argmax(landmark_list[:, 0]))

        left_up_point = np.int32([landmark_list[far_left][0], cut_h])
        left_down_point = np.int32([landmark_list[far_left][0], height])

        right_up_point = np.int32([landmark_list[far_right][0], cut_h])
        right_down_point = np.int32([landmark_list[far_right][0], height])
    else:
        left_up_point = np.int32([far_left, cut_h])
        left_down_point = np.int32([far_left, height])

        right_up_point = np.int32([far_right, cut_h])
        right_down_point = np.int32([far_right, height])

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    left_up_point,
                    left_down_point,
                    right_up_point,
                    right_down_point,
                    left_up_point,
                    right_up_point,
                    left_down_point,
                    right_down_point,
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    return mask


def face_mask_box_pose_big_cover_nose(img_shape, landmark_list, dtype="uint8"):
    height, width = img_shape[:2]

    middle_nose_point = 29

    cut_h = landmark_list[middle_nose_point, 1]

    mask = np.ones((height, width, 1), dtype=dtype)

    far_left = int(np.argmin(landmark_list[:, 0]))
    far_right = int(np.argmax(landmark_list[:, 0]))

    left_up_point = np.int32([landmark_list[far_left][0], cut_h])
    left_down_point = np.int32([landmark_list[far_left][0], height])

    right_up_point = np.int32([landmark_list[far_right][0], cut_h])
    right_down_point = np.int32([landmark_list[far_right][0], height])

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    left_up_point,
                    left_down_point,
                    right_up_point,
                    right_down_point,
                    left_up_point,
                    right_up_point,
                    left_down_point,
                    right_down_point,
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    return mask


def face_mask_square(img_shape, landmark_list, dtype="uint8"):
    height, width = img_shape[:2]

    mask = np.ones((height, width, 1), dtype=dtype)

    far_left = np.min(landmark_list[:, 0])
    far_right = np.max(landmark_list[:, 1])
    print("far_left {}, far_right {}".format(far_left, far_right))

    left_p = 2
    right_p = 14

    print(
        "left_p {}, right_p {}".format(
            landmark_list[left_p][0], landmark_list[right_p][0]
        )
    )

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    landmark_list[left_p],
                    [landmark_list[left_p][0], height],
                    landmark_list[right_p],
                    [landmark_list[right_p][0], height],
                    landmark_list[left_p],
                    landmark_list[right_p],
                    [landmark_list[left_p][0], height],
                    [landmark_list[right_p][0], height],
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    return mask


" Used for half face "


def bbox2mask(img_shape, bbox, dtype="uint8"):
    """Generate mask in ndarray from bbox.

    The returned mask has the shape of (h, w, 1). '1' indicates the
    hole and '0' indicates the valid regions.

    We prefer to use `uint8` as the data type of masks, which may be different
    from other codes in the community.

    Args:
            img_shape (tuple[int]): The size of the image.
            bbox (tuple[int]): Configuration tuple, (top, left, height, width)
            dtype (str): Indicate the data type of returned masks. Default: 'uint8'

    Return:
            numpy.ndarray: Mask in the shape of (h, w, 1).
    """

    height, width = img_shape[:2]

    mask = np.ones((height, width, 1), dtype=dtype)
    mask[bbox[0] : bbox[0] + bbox[2], bbox[1] : bbox[1] + bbox[3], :] = 0.0

    return mask


def face_mask_cheeks(img_shape, landmark_list, dtype="uint8"):
    height, width = img_shape[:2]
    mask = np.ones((height, width, 1), dtype=dtype)

    middle_nose_point = 29
    nose = 33
    cut_h = int(landmark_list[middle_nose_point, 1])

    far_left = int(np.argmin(landmark_list[:, 0]))
    far_right = int(np.argmax(landmark_list[:, 0]))

    left_up_point = np.int32([landmark_list[far_left][0], cut_h])
    left_down_point = np.int32([landmark_list[far_left][0], height])

    right_up_point = np.int32([landmark_list[far_right][0], cut_h])
    right_down_point = np.int32([landmark_list[far_right][0], height])

    cv2.drawContours(
        mask,
        np.int32(
            [
                [
                    left_up_point,
                    left_down_point,
                    right_up_point,
                    right_down_point,
                    left_up_point,
                    right_up_point,
                    left_down_point,
                    right_down_point,
                ]
            ]
        ),
        -1,
        color=(0),
        thickness=cv2.FILLED,
    )

    #  Calculate the bounding box coordinates for the nose
    nose_jaw_dist = (
        abs(landmark_list[2][0] - landmark_list[middle_nose_point][0]) * 0.10
    )  # 1, 15
    # nose_right_dist = (landmark_list[middle_nose_point][0] - landmark_list[1][0]) * 0.10
    # nose_left_dist = (landmark_list[15][0] - landmark_list[middle_nose_point][0]) * 0.10
    #

    nose_min_x = int(landmark_list[31][0] - nose_jaw_dist)
    nose_max_x = int(landmark_list[35][0] + nose_jaw_dist)
    # nose_min_x = int(landmark_list[31][0] - nose_right_dist)
    # nose_max_x = int(landmark_list[35][0] + nose_left_dist)
    nose_min_y = cut_h
    nose_max_y = int(landmark_list[nose, 1])

    # Clear the nose area from the mask using a rectangle
    mask_nose = np.ones((height, width, 1), dtype=dtype)
    cv2.rectangle(
        mask_nose,
        (nose_min_x, nose_min_y),
        (nose_max_x, nose_max_y),
        color=(0),
        thickness=cv2.FILLED,
    )

    mask_nose = 1 - mask_nose
    mask = mask + mask_nose

    return mask


def face_mask_cheeks_batch(
    img_shape, landmark_list, dtype="uint8", box_expand=0.0, show_nose=True
):
    height, width = img_shape[:2]

    # Handle both single and multiple landmarks
    if len(landmark_list.shape) == 2:
        landmark_list = landmark_list[None, ...]  # Add batch dimension
    num_frames = landmark_list.shape[0]

    # Initialize masks for all frames
    masks = np.ones((num_frames, height, width), dtype=dtype)

    for i in range(num_frames):
        landmarks = landmark_list[i]
        middle_nose_point = 29
        nose = 33
        cut_h = int(landmarks[middle_nose_point, 1])

        # Add height expansion
        if box_expand > 0:
            cut_h = max(0, cut_h - int(box_expand * height))

        far_left = int(np.argmin(landmarks[:, 0]))
        far_right = int(np.argmax(landmarks[:, 0]))

        left_up_point = np.int32([landmarks[far_left][0], cut_h])
        left_down_point = np.int32([landmarks[far_left][0], height])

        right_up_point = np.int32([landmarks[far_right][0], cut_h])
        right_down_point = np.int32([landmarks[far_right][0], height])

        cv2.drawContours(
            masks[i],
            np.int32(
                [
                    [
                        left_up_point,
                        left_down_point,
                        right_up_point,
                        right_down_point,
                        left_up_point,
                        right_up_point,
                        left_down_point,
                        right_down_point,
                    ]
                ]
            ),
            -1,
            color=(0),
            thickness=cv2.FILLED,
        )

        if show_nose:
            #  Calculate the bounding box coordinates for the nose
            nose_jaw_dist = (
                abs(landmarks[2][0] - landmarks[middle_nose_point][0]) * 0.10
            )  # 1, 15

            nose_min_x = int(landmarks[31][0] - nose_jaw_dist)
            nose_max_x = int(landmarks[35][0] + nose_jaw_dist)
            nose_min_y = cut_h
            nose_max_y = int(landmarks[nose, 1])

            # Clear the nose area from the mask using a rectangle
            mask_nose = np.ones((height, width), dtype=dtype)
            cv2.rectangle(
                mask_nose,
                (nose_min_x, nose_min_y),
                (nose_max_x, nose_max_y),
                color=(0),
                thickness=cv2.FILLED,
            )

            mask_nose = 1 - mask_nose
            masks[i] = masks[i] + mask_nose

    # If input was single frame, return single mask
    if landmark_list.shape[0] == 1:
        return masks[0]

    return 1 - torch.from_numpy(masks)