import gradio as gr from evaluate import eval, install from datetime import datetime import os, shutil, spaces import warnings, tempfile from audio import gen_audio warnings.filterwarnings("ignore") warnings.simplefilter("ignore") import time, cv2, os videos = { # "教學視頻1": "teach/00.mp4", # "教學視頻2": "teach/01.mp4", # "教學視頻3": "teach/02.mp4", # "教學視頻4": "teach/03.mp4", "(病症)高峰期": "teach/(病症)高峰期.mp4", "公園": "teach/公園.mp4", "健康": "teach/健康.mp4", "刷牙": "teach/刷牙.mp4", "圖書館": "teach/圖書館.mp4", "下午": "teach/下午.mp4", "中風": "teach/中風.mp4", "中暑": "teach/中暑.mp4", "主題": "teach/主題.mp4", "住院證明": "teach/住院證明.mp4" } jsons = { # "教學視頻1": "teach/00.json", # "教學視頻2": "teach/01.json", # "教學視頻3": "teach/02.json", # "教學視頻4": "teach/03.json", "(病症)高峰期": "teach/(病症)高峰期.json", "公園": "teach/公園.json", "健康": "teach/健康.json", "刷牙": "teach/刷牙.json", "圖書館": "teach/圖書館.json", "下午": "teach/下午.json", "中風": "teach/中風.json", "中暑": "teach/中暑.json", "主題": "teach/主題.json", "住院證明": "teach/住院證明.json" } def get_video_url(option): # print(videos[option]) return videos[option] def flip_video_horizontally(video_path): if not os.path.exists(video_path): print(f"Error: {video_path} does not exist.") return cap = cv2.VideoCapture(video_path) if not cap.isOpened(): print(f"Error: cannot open {video_path}.") return frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(cap.get(cv2.CAP_PROP_FPS)) fourcc = cv2.VideoWriter_fourcc(*'XVID') #int(cap.get(cv2.CAP_PROP_FOURCC)) temp_output_path = video_path + "_temp.mp4" out = cv2.VideoWriter(temp_output_path, fourcc, fps, (frame_width, frame_height)) while True: ret, frame = cap.read() if not ret: break flipped_frame = cv2.flip(frame, 1) out.write(flipped_frame) cap.release() out.release() print(video_path, os.path.exists(video_path)) print(temp_output_path, os.path.exists(temp_output_path)) os.replace(temp_output_path, video_path) print(f"Finished flipping video:{video_path}") @spaces.GPU() def evaluate_sign_language(user_video, standard_video_option): if user_video.split('/')[-1].startswith("sample."): flip_video_horizontally(user_video) tmp = tempfile.TemporaryDirectory() tmpdir = tmp.name new_path = tmpdir + "/user.mp4" shutil.copy(user_video, new_path) shutil.copy(videos[standard_video_option], tmpdir + "/standard.mp4") shutil.copy(jsons[standard_video_option], tmpdir + "/standard.json") print("Copy User Video to:", new_path) print("Copy Standard Video to:", tmpdir + "/standard.mp4") print("Copy Standard JSON to:", tmpdir + "/standard.json") test, standard = user_video, videos[standard_video_option] score, final_merged_intervals, comments = eval(test, standard, tmpdir) if score < 95: score = 0 elif score < 98: score = 50 * (score - 95) / 3 elif 98 <= score <= 100: score = 50 * (score - 98) / 2 + 50 current_date = datetime.now().strftime("%Y年%m月%d日") # 初始化反馈报告 qualification = "不合格" if score < 60 else "合格" advice = f"""
\n手語表現反饋報告



日期: {current_date}

改進建議(供參考):
""" part_translation = { 'Right Hand': '右手細節、', 'Left Hand': '左手細節、', 'Right Arm': '右臂、', 'Left Arm': '左臂、' } # 为每个问题区间生成具体观察结果 f = 0 subtitles = [] for interval, errors in final_merged_intervals: advice += f" - 時間區間:{interval[0]}-{interval[1]}秒
" parts = "" for error in errors: part = part_translation[error] parts += part advice += f" 觀察結果:您的{parts[:-1]}動作與標準手語手勢不太一致。
" f = 1 subtitles.append([[interval[0], interval[1]], parts[:-1]]) if not f: advice = advice[:-4] advice += "無" advice += "
建議:請參考標準視頻,模仿正確的動作,保持速度一致。

" advice += f"整體结果: {qualification}
" # advice += f"整體结果: 您本次的總評分為 {score:.2f}/100分。
" ret_video = gen_audio(subtitles, tmpdir) return advice, ret_video, tmp def cleanup_temp_dir(tmp_dir): if tmp_dir: time.sleep(0.5) print(f"Cleaning up temporary directory: {tmp_dir.name}") tmp_dir.cleanup() return "临时目录已清理" return "没有临时目录需要清理" install() font = ["Heiti SC", "FangSong"] title = """

手語教學與評估系統

""" with gr.Blocks( css=""" .contain { display: flex; flex-direction: column; } .gradio-container { height: 200vh !important; overflow-y: auto; /* 添加垂直滚动条 */ } #col_container { height: 100%; } pre { white-space: pre-wrap; /* Since CSS 2.1 */ white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ white-space: -pre-wrap; /* Opera 4-6 */ white-space: -o-pre-wrap; /* Opera 7 */ word-wrap: break-word; /* Internet Explorer 5.5+ */ }""", js=""" function refresh() { const url = new URL(window.location); if (url.searchParams.get('__theme') !== 'light') { url.searchParams.set('__theme', 'light'); window.location.href = url.href; } }""", title="手語教學與評估系統", theme=gr.themes.Soft(), ) as app: gr.HTML(title) with gr.Row(): with gr.Column(): video_selector = gr.Dropdown(list(videos.keys()), label="請選擇教學視頻") video_player = gr.Video(label="請觀看教學視頻演示") with gr.Column(): upload_video = gr.Video(label="請錄製/上傳您的視頻,點擊\'開始評估\'") evaluate_button = gr.Button("開始評估") with gr.Row(): with gr.Column(): advice_output = gr.HTML(label="反饋報告") with gr.Column(): compare_video_player = gr.Video(label="評估結果") video_selector.change(get_video_url, inputs=video_selector, outputs=video_player) tmp_dir_state = gr.State() evaluate_button.click( evaluate_sign_language, inputs=[upload_video, video_selector], outputs=[advice_output, compare_video_player, tmp_dir_state], ) compare_video_player.change( fn=cleanup_temp_dir, inputs=tmp_dir_state, outputs=None ) app.launch(share=True)