demo/mask_utils.py

import cv2
from PIL import Image
import numpy as np
import torch
import gradio as gr

class ImageSketcher(gr.Image):
    """
    Code is from https://github.com/jshilong/GPT4RoI/blob/7c157b5f33914f21cfbc804fb301d3ce06324193/gpt4roi/app.py#L365

    Fix the bug of gradio.Image that cannot upload with tool == 'sketch'.
    """

    is_template = True  # Magic to make this work with gradio.Block, don't remove unless you know what you're doing.

    def __init__(self, **kwargs):
        super().__init__(tool='boxes', **kwargs)

    def preprocess(self, x):
        if x is None:
            return x
        if self.tool == 'boxes' and self.source in ['upload', 'webcam']:
            if isinstance(x, str):
                x = {'image': x, 'boxes': []}
            else:
                assert isinstance(x, dict)
                assert isinstance(x['image'], str)
                assert isinstance(x['boxes'], list)
        x = super().preprocess(x)
        return x
    
def process_mask_to_show(mask):
    '''
        Process the mask to show on the gradio.Image
    '''
    mask = np.array(mask > 0.1, dtype=np.uint8) * 255
    mask_stacked = np.stack([mask] * 3, axis=-1)

    return mask_stacked

def img_add_masks(img_, colored_mask, mask, linewidth=2):
    if type(img_) is np.ndarray:
        img = Image.fromarray(img_, mode='RGB').convert('RGBA')
    else:
        img = img_.copy()
    h, w = img.height, img.width
    # contour
    temp = np.zeros((h, w, 1))
    contours, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cv2.drawContours(temp, contours, -1, (255, 255, 255), linewidth)
    color = np.array([1, 1, 1, 1])
    contour_mask = temp * color.reshape(1, 1, -1)

    overlay_inner = Image.fromarray(colored_mask.astype(np.uint8), 'RGBA')
    img.paste(overlay_inner, (0, 0), overlay_inner)

    overlay_contour = Image.fromarray(contour_mask.astype(np.uint8), 'RGBA')
    img.paste(overlay_contour, (0, 0), overlay_contour)
    return img

def gen_colored_masks(
        annotation,
        random_color=False,
):
    """
    Code is largely based on https://github.com/CASIA-IVA-Lab/FastSAM/blob/4d153e909f0ad9c8ecd7632566e5a24e21cf0071/utils/tools_gradio.py#L130
    """
    device = annotation.device
    mask_sum = annotation.shape[0]
    height = annotation.shape[1]
    weight = annotation.shape[2]
    areas = torch.sum(annotation, dim=(1, 2))
    sorted_indices = torch.argsort(areas, descending=False)
    annotation = annotation[sorted_indices]

    index = (annotation != 0).to(torch.long).argmax(dim=0)
    if random_color:
        color = torch.rand((mask_sum, 1, 1, 3)).to(device)
    else:
        color = torch.ones((mask_sum, 1, 1, 3)).to(device) * torch.tensor(
            [30 / 255, 144 / 255, 255 / 255]
        ).to(device)
    transparency = torch.ones((mask_sum, 1, 1, 1)).to(device) * 0.6
    visual = torch.cat([color, transparency], dim=-1)
    mask_image = torch.unsqueeze(annotation, -1) * visual

    mask = torch.zeros((height, weight, 4)).to(device)
    h_indices, w_indices = torch.meshgrid(torch.arange(height), torch.arange(weight))
    indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None))

    mask[h_indices, w_indices, :] = mask_image[indices]
    mask_cpu = mask.cpu().numpy()

    return mask_cpu, sorted_indices

def mask_foreground(mask, trans=0.6, random_color=True):
    if random_color:
        color = np.concatenate([np.random.random(3) * 255, np.array([trans * 255])], axis=0)
    else:
        color = np.array([30, 144, 255, trans * 255])
    h, w = mask.shape[-2:]
    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)

    return mask_image


def mask_background(mask, trans=0.5):
    h, w = mask.shape[-2:]
    mask_image = (1 - mask.reshape(h, w, 1)) * np.array([0, 0, 0, trans * 255])

    return mask_image


def mask_select_point(all_masks, output_mask_2_raw, mask_order, evt: gr.SelectData):
    h, w = output_mask_2_raw.height, output_mask_2_raw.width
    pointed_mask = None
    for i in range(len(mask_order)):
        idx = mask_order[i]
        msk = all_masks[idx]
        if msk[evt.index[1], evt.index[0]] == 1:
            pointed_mask = msk.copy()
            break

    if pointed_mask is not None:
        contours, hierarchy = cv2.findContours(pointed_mask.astype("uint8"), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        ret = output_mask_2_raw.copy()

        temp = np.zeros((h, w, 1))
        contours, _ = cv2.findContours(msk.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        cv2.drawContours(temp, contours, -1, (255, 255, 255), 3)
        color = np.array([1, 1, 1, 1])
        contour_mask = temp * color.reshape(1, 1, -1)

        colored_mask = mask_background(pointed_mask)

        overlay_inner = Image.fromarray(colored_mask.astype(np.uint8), 'RGBA')
        ret.paste(overlay_inner, (0, 0), overlay_inner)

        overlay_contour = Image.fromarray(contour_mask.astype(np.uint8), 'RGBA')
        ret.paste(overlay_contour, (0, 0), overlay_contour)

        return ret, pointed_mask
    else:
        return output_mask_2_raw, None