app.py

import gradio as gr
import cv2
import os
import mediapipe as mp
import numpy as np
import tempfile
from maps import *
from functions import *

class CosmeticInjectionVisualizer:
    def __init__(self, muscles_map, tasks_map):
        self.mp_face_mesh = mp.solutions.face_mesh
        self.mp_drawing = mp.solutions.drawing_utils
        self.mp_drawing_styles = mp.solutions.drawing_styles
        self.muscles_map = muscles_map
        self.tasks_map = tasks_map

    def process_image(self, image, task_name):
        frame_shape = image.shape
        with self.mp_face_mesh.FaceMesh(
            static_image_mode=True,
            refine_landmarks=True,
            max_num_faces=1,
            min_detection_confidence=0.5) as face_mesh:

            results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
            if results.multi_face_landmarks:
                for face_landmarks in results.multi_face_landmarks:
                    points = self.get_muscle_points(task_name, face_landmarks, frame_shape)
                    self.draw_muscle_points(image, points, face_landmarks, task_name)
        return image

    def draw_rounded_rectangle(self,image, start_point, end_point, color, thickness, radius):
      top_left = start_point
      bottom_right = end_point

      if thickness < 0:  # Filled rectangle
          thickness = cv2.FILLED

      # Draw filled rectangle with rounded corners
      if thickness == cv2.FILLED:
          # Top-left corner
          cv2.ellipse(image, (top_left[0] + radius, top_left[1] + radius), (radius, radius), 180, 0, 90, color, -1)
          # Top-right corner
          cv2.ellipse(image, (bottom_right[0] - radius, top_left[1] + radius), (radius, radius), 270, 0, 90, color, -1)
          # Bottom-left corner
          cv2.ellipse(image, (top_left[0] + radius, bottom_right[1] - radius), (radius, radius), 90, 0, 90, color, -1)
          # Bottom-right corner
          cv2.ellipse(image, (bottom_right[0] - radius, bottom_right[1] - radius), (radius, radius), 0, 0, 90, color, -1)

          # Top and bottom border
          cv2.rectangle(image, (top_left[0] + radius, top_left[1]), (bottom_right[0] - radius, top_left[1] + radius), color, -1)
          cv2.rectangle(image, (top_left[0] + radius, bottom_right[1] - radius), (bottom_right[0] - radius, bottom_right[1]), color, -1)
          # Left and right border
          cv2.rectangle(image, (top_left[0], top_left[1] + radius), (top_left[0] + radius, bottom_right[1] - radius), color, -1)
          cv2.rectangle(image, (bottom_right[0] - radius, top_left[1] + radius), (bottom_right[0], bottom_right[1] - radius), color, -1)
          # Center rectangle
          cv2.rectangle(image, (top_left[0] + radius, top_left[1] + radius), (bottom_right[0] - radius, bottom_right[1] - radius), color, -1)
      else:
          # Top-left corner
          cv2.ellipse(image, (top_left[0] + radius, top_left[1] + radius), (radius, radius), 180, 0, 90, color, thickness)
          # Top-right corner
          cv2.ellipse(image, (bottom_right[0] - radius, top_left[1] + radius), (radius, radius), 270, 0, 90, color, thickness)
          # Bottom-left corner
          cv2.ellipse(image, (top_left[0] + radius, bottom_right[1] - radius), (radius, radius), 90, 0, 90, color, thickness)
          # Bottom-right corner
          cv2.ellipse(image, (bottom_right[0] - radius, bottom_right[1] - radius), (radius, radius), 0, 0, 90, color, thickness)

          # Top border
          cv2.line(image, (top_left[0] + radius, top_left[1]), (bottom_right[0] - radius, top_left[1]), color, thickness)
          # Bottom border
          cv2.line(image, (top_left[0] + radius, bottom_right[1]), (bottom_right[0] - radius, bottom_right[1]), color, thickness)
          # Left border
          cv2.line(image, (top_left[0], top_left[1] + radius), (top_left[0], bottom_right[1] - radius), color, thickness)
          # Right border
          cv2.line(image, (bottom_right[0], top_left[1] + radius), (bottom_right[0], bottom_right[1] - radius), color, thickness)

    def draw_muscle_points(self, image, points, face_landmarks, task, draw_background=False, verbose=False):
        # Calculate bounding box of the face landmarks
        x_min = min([landmark.x for landmark in face_landmarks.landmark]) * image.shape[1]
        y_min = min([landmark.y for landmark in face_landmarks.landmark]) * image.shape[0]
        x_max = max([landmark.x for landmark in face_landmarks.landmark]) * image.shape[1]
        y_max = max([landmark.y for landmark in face_landmarks.landmark]) * image.shape[0]
        face_width = x_max - x_min
        face_height = y_max - y_min

        # Determine text size and circle size based on face size relative to the image
        scale_factor = 0.0005 * (face_width + face_height)
        text_scale = scale_factor
        thickness = int(2 * scale_factor)
        margin = int(10 * scale_factor)
        circle_radius = int(5 * scale_factor)
        radius = int(10 * scale_factor)  # For rounded corners

        muscle_names = set()
        for (x, y, muscle) in points:
            # Draw the circle for the muscle point
            cv2.circle(image, (x, y), circle_radius, (0, 0, 255), -1)

            # Determine text size and background size
            text = "3 U"
            (text_width, text_height), baseline = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, text_scale, thickness)
            text_x = x
            text_y = y - 10
            text_x_end = text_x + text_width + 2 * margin
            text_y_end = text_y - text_height - 2 * margin

            if draw_background:
                # Draw background rectangle with margins and rounded corners
                self.draw_rounded_rectangle(image, (text_x - margin, text_y + margin), (text_x_end, text_y_end), (0, 0, 0), cv2.FILLED, radius)

            if verbose:
              # Draw the text on top of the rectangle
              cv2.putText(image, text, (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, text_scale, (0, 255, 0), thickness, cv2.LINE_AA)

            muscle_names.add(muscle)
        muscles = ','.join(muscle_names)
        # Draw legend in the bottom-left corner with background
        legend_texts = ['Total dose : 42','Name of patient: Julia Juila',f'Muscle : {muscles}', f'Task : {task}','Cosmetic App']
        legend_x = 10
        legend_y = image.shape[0] - 10
        legend_margin = 5
        legend_scale_factor = 1.5 * text_scale  # Make legend text larger
        legend_thickness = int(2 * legend_scale_factor)
        legend_radius = int(10 * legend_scale_factor)
        max_text_width = 0
        total_text_height = 0

        for text in legend_texts:
            (text_width, text_height), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, legend_scale_factor, legend_thickness)
            max_text_width = max(max_text_width, text_width)
            total_text_height += text_height + 2 * legend_margin

        legend_start_point = (legend_x - legend_margin, legend_y - total_text_height - legend_margin)
        legend_end_point = (legend_x + max_text_width + legend_margin, legend_y + legend_margin)

        if True:
            self.draw_rounded_rectangle(image, legend_start_point, legend_end_point, (0, 0, 0), cv2.FILLED, legend_radius)

        for text in legend_texts:
            (text_width, text_height), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, legend_scale_factor, legend_thickness)
            cv2.putText(image, text, (legend_x, legend_y), cv2.FONT_HERSHEY_SIMPLEX, legend_scale_factor, (255, 255, 255), legend_thickness, cv2.LINE_AA)
            legend_y -= text_height + 2 * legend_margin


    def get_muscle_points(self, task_name, face_landmarks, frame_shape):
        if task_name not in self.tasks_map:
            raise ValueError(f"Task '{task_name}' not found in tasks map.")
        muscles_names = self.tasks_map[task_name]['muscles']
        points = []
        for muscle in muscles_names:
            for region in self.muscles_map[muscle]:
                if 'points' in self.muscles_map[muscle][region]:
                    for point_idx in self.muscles_map[muscle][region]['points']:
                        landmark = face_landmarks.landmark[point_idx]
                        x = int(landmark.x * frame_shape[1])
                        y = int(landmark.y * frame_shape[0])
                        points.append((x, y, muscle))
                else:
                    for subregion in self.muscles_map[muscle][region]:
                        for point_idx in self.muscles_map[muscle][region][subregion]['points']:
                            landmark = face_landmarks.landmark[point_idx]
                            x = int(landmark.x * frame_shape[1])
                            y = int(landmark.y * frame_shape[0])
                            points.append((x, y, muscle))
        return points

    def process_video(self, video_path, task_name):
        cap = cv2.VideoCapture(video_path)
        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4')
        output_path = temp_file.name

        # Get the width and height of the frames
        frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
        frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = int(cap.get(cv2.CAP_PROP_FPS))

        # Define the codec and create VideoWriter object
        fourcc = cv2.VideoWriter_fourcc(*'mp4v')  # Use appropriate codec for .mp4 files
        out = cv2.VideoWriter(output_path, fourcc, fps, (frame_width, frame_height))

        with self.mp_face_mesh.FaceMesh(
            static_image_mode=False,
            refine_landmarks=True,
            max_num_faces=1,
            min_detection_confidence=0.5,
            min_tracking_confidence=0.5) as face_mesh:

            while cap.isOpened():
                ret, frame = cap.read()
                if not ret:
                    break
                frame_shape = frame.shape
                results = face_mesh.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
                if results.multi_face_landmarks:
                    for face_landmarks in results.multi_face_landmarks:
                        points = self.get_muscle_points(task_name, face_landmarks, frame_shape)
                        self.draw_muscle_points(frame, points, face_landmarks, task_name)
                
                # Write the processed frame to the output video file
                out.write(frame)

            cap.release()
            out.release()
            cv2.destroyAllWindows()

        return output_path

    def process_webcam(self, task_name):
        cap = cv2.VideoCapture(0)
        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4')
        output_path = temp_file.name

        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
        out = cv2.VideoWriter(output_path, fourcc, 20.0, (640, 480))
        
        with self.mp_face_mesh.FaceMesh(
            static_image_mode=False,
            refine_landmarks=True,
            max_num_faces=1,
            min_detection_confidence=0.5,
            min_tracking_confidence=0.5) as face_mesh:

            while cap.isOpened():
                ret, frame = cap.read()
                if not ret:
                    break
                frame_shape = frame.shape
                results = face_mesh.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
                if results.multi_face_landmarks:
                    for face_landmarks in results.multi_face_landmarks:
                        points = self.get_muscle_points(task_name, face_landmarks, frame_shape)
                        self.draw_muscle_points(frame, points, face_landmarks, task_name)
                out.write(frame)

            cap.release()
            out.release()
            cv2.destroyAllWindows()

        return output_path

visualizer = CosmeticInjectionVisualizer(muscles_map, tasks_map)

def inference_image(image, task_name):
    result_image = visualizer.process_image(image, task_name)
    return result_image

def inference_video(video_path, task_name):
    result_video_path = visualizer.process_video(video_path, task_name)
    return result_video_path

def inference_webcam(task_name):
    result_video_path = visualizer.process_webcam(task_name)
    return result_video_path

task_names = list(tasks_map.keys())
base_path=os.getcwd()
default_image_path = os.path.join(base_path,'image.jpg')
default_video_path = os.path.join(base_path,'video.mp4')

with gr.Blocks() as demo:
    gr.Markdown("# Cosmetic Injection Visualizer")
    
    with gr.Tabs():
        with gr.TabItem("Image"):
            image_input = gr.Image(type="numpy", label="Input Image", value=default_image_path)
            task_input_image = gr.Dropdown(choices=task_names, label="Task Name")
            image_output = gr.Image(type="numpy", label="Output Image")
            gr.Button("Process Image").click(inference_image, inputs=[image_input, task_input_image], outputs=image_output)

        with gr.TabItem("Video"):
            video_input = gr.Video(label="Input Video", value=default_video_path)
            task_input_video = gr.Dropdown(choices=task_names, label="Task Name")
            video_output = gr.Video(label="Output Video")
            gr.Button("Process Video").click(inference_video, inputs=[video_input, task_input_video], outputs=video_output)

        with gr.TabItem("Webcam"):
            task_input_webcam = gr.Dropdown(choices=task_names, label="Task Name")
            webcam_output = gr.Video(label="Output Video")
            gr.Button("Process Webcam").click(inference_webcam, inputs=[task_input_webcam], outputs=webcam_output)

demo.launch()