app.py

import gradio as gr
from evaluate import eval, install
from datetime import datetime
import os, shutil, spaces
import warnings, tempfile
from audio import gen_audio
warnings.filterwarnings("ignore")
warnings.simplefilter("ignore")
import time, cv2, os

videos = {
    # "教學視頻1": "teach/00.mp4",
    # "教學視頻2": "teach/01.mp4",
    # "教學視頻3": "teach/02.mp4",
    # "教學視頻4": "teach/03.mp4",
    "(病症)高峰期": "teach/(病症)高峰期.mp4",
    "公園": "teach/公園.mp4",
    "健康": "teach/健康.mp4",
    "刷牙": "teach/刷牙.mp4",
    "圖書館": "teach/圖書館.mp4",
    "下午": "teach/下午.mp4",
    "中風": "teach/中風.mp4",
    "中暑": "teach/中暑.mp4",
    "主題": "teach/主題.mp4",
    "住院證明": "teach/住院證明.mp4"
}

jsons = {
    # "教學視頻1": "teach/00.json",
    # "教學視頻2": "teach/01.json",
    # "教學視頻3": "teach/02.json",
    # "教學視頻4": "teach/03.json",
    "(病症)高峰期": "teach/(病症)高峰期.json",
    "公園": "teach/公園.json",
    "健康": "teach/健康.json",
    "刷牙": "teach/刷牙.json",
    "圖書館": "teach/圖書館.json",
    "下午": "teach/下午.json",
    "中風": "teach/中風.json",
    "中暑": "teach/中暑.json",
    "主題": "teach/主題.json",
    "住院證明": "teach/住院證明.json"
}

def get_video_url(option):
    # print(videos[option])
    return videos[option]

def flip_video_horizontally(video_path):
    if not os.path.exists(video_path):
        print(f"Error: {video_path} does not exist.")
        return

    cap = cv2.VideoCapture(video_path)
    if not cap.isOpened():
        print(f"Error: cannot open {video_path}.")
        return

    frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
    frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    fps = int(cap.get(cv2.CAP_PROP_FPS))
    fourcc = int(cap.get(cv2.CAP_PROP_FOURCC))

    temp_output_path = video_path + "_temp.mp4"

    out = cv2.VideoWriter(temp_output_path, fourcc, fps, (frame_width, frame_height))

    while True:
        ret, frame = cap.read()
        if not ret:
            break
        flipped_frame = cv2.flip(frame, 1)
        out.write(flipped_frame)

    cap.release()
    out.release()

    os.replace(temp_output_path, video_path)
    print(f"Finished flipping video：{video_path}")
    
@spaces.GPU()
def evaluate_sign_language(user_video, standard_video_option):
    if user_video.split('/')[-1].startswith("sample."):
        flip_video_horizontally(user_video)
        
    tmp = tempfile.TemporaryDirectory()
    tmpdir = tmp.name
    new_path = tmpdir + "/user.mp4"
    shutil.copy(user_video, new_path)
    shutil.copy(videos[standard_video_option], tmpdir + "/standard.mp4")
    shutil.copy(jsons[standard_video_option], tmpdir + "/standard.json")
    print("Copy User Video to:", new_path)
    print("Copy Standard Video to:", tmpdir + "/standard.mp4")
    print("Copy Standard JSON to:", tmpdir + "/standard.json")

    test, standard = user_video, videos[standard_video_option]
    score, final_merged_intervals, comments = eval(test, standard, tmpdir)

    if score < 95:
        score = 0
    elif score < 98:
        score = 50 * (score - 95) / 3  
    elif 98 <= score <= 100:
        score = 50 * (score - 98) / 2 + 50

    current_date = datetime.now().strftime("%Y年%m月%d日")

    # 初始化反馈报告
    qualification = "不合格" if score < 60 else "合格"
    advice = f"""<div style='font-family: "Heiti SC"; text-align: center; font-size: 24px;'><b>\n手語表現反饋報告</b></div>
    <br><br><br><b>日期：</b> {current_date}<br><br><b>改進建議（供參考）：</b><br>"""
    
    part_translation = {
        'Right Hand': '右手細節、',
        'Left Hand': '左手細節、',
        'Right Arm': '右臂、',
        'Left Arm': '左臂、'
    }

    # 为每个问题区间生成具体观察结果
    f = 0
    subtitles = []

    for interval, errors in final_merged_intervals:
        advice += f"<b> - 時間區間：{interval[0]}-{interval[1]}秒</b><br>"
        parts = ""
        for error in errors:
            part = part_translation[error]
            parts += part
        advice += f"   觀察結果：您的{parts[:-1]}動作與標準手語手勢不太一致。<br>"
        f = 1
        subtitles.append([[interval[0], interval[1]], parts[:-1]])

    if not f:   
        advice = advice[:-4]
        advice += "無"

    advice += "<br>建議：請參考標準視頻，模仿正確的動作，保持速度一致。<br><br>"
    
    advice += f"<b>整體结果：</b> {qualification}<br>"
    # advice += f"<b>整體结果：</b> 您本次的總評分為 {score:.2f}/100分。<br>"

    ret_video = gen_audio(subtitles, tmpdir)
    return advice, ret_video, tmp

def cleanup_temp_dir(tmp_dir):
    if tmp_dir:
        time.sleep(0.5)
        print(f"Cleaning up temporary directory: {tmp_dir.name}")
        tmp_dir.cleanup()
        return "临时目录已清理"
    return "没有临时目录需要清理"

install()
font = ["Heiti SC", "FangSong"]
title = """<h1 style="text-align: center;">
<div style="width: 1.4em; height: 1.4em; display: inline-block;"><img src="https://github.com/ZiyuGuo99/ZiyuGuo99.github.io/blob/main/assets/img/sl.png?raw=true" style='width: 100%; height: 100%; object-fit: contain;' /></div>
<span style="font-family: """ + font[0] + """; font-variant: small-caps; font-weight: bold;">手語教學與評估系統</span>
</h1>
"""
with gr.Blocks(
       css="""
       .contain { display: flex; flex-direction: column; }
       .gradio-container { height: 200vh !important; 
       overflow-y: auto; /* 添加垂直滚动条 */
                    }
       #col_container { height: 100%; }
       pre {
       white-space: pre-wrap;       /* Since CSS 2.1 */
       white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
       white-space: -pre-wrap;      /* Opera 4-6 */
       white-space: -o-pre-wrap;    /* Opera 7 */
       word-wrap: break-word;       /* Internet Explorer 5.5+ */
       }""",
       js="""
       function refresh() {
           const url = new URL(window.location);
           if (url.searchParams.get('__theme') !== 'light') {
               url.searchParams.set('__theme', 'light');
               window.location.href = url.href;
           }
       }""",
       title="手語教學與評估系統",
       theme=gr.themes.Soft(),
   ) as app:
    gr.HTML(title)
    with gr.Row():
        with gr.Column():
            video_selector = gr.Dropdown(list(videos.keys()), label="請選擇教學視頻")
            video_player = gr.Video(label="請觀看教學視頻演示")
        with gr.Column():
            upload_video = gr.Video(label="請錄製/上傳您的視頻，點擊\'開始評估\'") 
            evaluate_button = gr.Button("開始評估")
    with gr.Row():
        with gr.Column():
            advice_output = gr.HTML(label="反饋報告")
        with gr.Column():
            compare_video_player = gr.Video(label="評估結果")
    video_selector.change(get_video_url, inputs=video_selector, outputs=video_player)
    tmp_dir_state = gr.State()
    evaluate_button.click(
        evaluate_sign_language, 
        inputs=[upload_video, video_selector], 
        outputs=[advice_output, compare_video_player, tmp_dir_state],
    )
    
    compare_video_player.change(
        fn=cleanup_temp_dir, 
        inputs=tmp_dir_state,  
        outputs=None
    )

app.launch(share=True)