Spaces:
Running
Running
| import gradio as gr | |
| import os | |
| import cv2 | |
| import face_recognition | |
| from fastai.vision.all import load_learner | |
| import time | |
| import base64 | |
| from deepface import DeepFace | |
| import torchaudio | |
| import moviepy.editor as mp | |
| from transformers import WhisperProcessor, WhisperForConditionalGeneration, pipeline | |
| # import pathlib | |
| # temp = pathlib.PosixPath | |
| # pathlib.PosixPath = pathlib.WindowsPath | |
| backends = [ | |
| 'opencv', | |
| 'ssd', | |
| 'dlib', | |
| 'mtcnn', | |
| 'retinaface', | |
| 'mediapipe' | |
| ] | |
| emotion_pipeline = pipeline("text-classification", model="j-hartmann/emotion-english-distilroberta-base", return_all_scores=True) | |
| sentiment_pipeline = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english") | |
| model = load_learner("gaze-recognizer-v3.pkl") | |
| def analyze_emotion(text): | |
| result = emotion_pipeline(text) | |
| return result | |
| def analyze_sentiment(text): | |
| result = sentiment_pipeline(text) | |
| return result | |
| def getTranscription(path): | |
| # Insert Local Video File Path | |
| clip = mp.VideoFileClip(path) | |
| # Insert Local Audio File Path | |
| clip.audio.write_audiofile(r"audio.wav") | |
| waveform, sample_rate = torchaudio.load("audio.wav") | |
| resampler = torchaudio.transforms.Resample(sample_rate, 16000) | |
| waveform = resampler(waveform)[0] | |
| processor = WhisperProcessor.from_pretrained("openai/whisper-tiny") | |
| model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny") | |
| model.config.forced_decoder_ids = None | |
| input_features = processor(waveform.squeeze(dim=0), return_tensors="pt").input_features | |
| predicted_ids = model.generate(input_features) | |
| transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True) | |
| return transcription[0] | |
| def process_frame(frame): | |
| # Convert the frame to RGB color (face_recognition uses RGB) | |
| gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) | |
| # Find all the faces in the frame using a pre-trained convolutional neural network. | |
| face_locations = face_recognition.face_locations(gray) | |
| #face_locations = face_recognition.face_locations(gray, number_of_times_to_upsample=0, model="cnn") | |
| if len(face_locations) > 0: | |
| # Show the original frame with face rectangles drawn around the faces | |
| for top, right, bottom, left in face_locations: | |
| # cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2) | |
| face_image = gray[top:bottom, left:right] | |
| color_image = frame[top:bottom, left:right] | |
| # Resize the face image to the desired size | |
| resized_face_image = cv2.resize(face_image, (128,128)) | |
| try: | |
| emotion = DeepFace.analyze(color_image,actions=['emotion'],detector_backend = backends[2],enforce_detection = False)# 2,3, 4 works | |
| emotion_count += 1 | |
| print(emotion) | |
| except Exception as e: | |
| emotion = 0 | |
| pass | |
| # Predict the class of the resized face image using the model | |
| result = model.predict(resized_face_image) | |
| print(result[0]) | |
| return result[0], emotion | |
| def video_processing(video_file, encoded_video): | |
| angry = 0 | |
| disgust = 0 | |
| fear = 0 | |
| happy = 0 | |
| sad = 0 | |
| surprise = 0 | |
| neutral = 0 | |
| emotion_count = 0 | |
| if encoded_video != "": | |
| decoded_file_data = base64.b64decode(encoded_video) | |
| with open("temp_video.mp4", "wb") as f: | |
| f.write(decoded_file_data) | |
| video_file = "temp_video.mp4" | |
| start_time = time.time() | |
| transcription = getTranscription(video_file) | |
| print(transcription) | |
| text_emotion = analyze_emotion(transcription) | |
| print(text_emotion) | |
| text_sentiment = analyze_sentiment(transcription) | |
| print(text_sentiment) | |
| video_capture = cv2.VideoCapture(video_file) | |
| on_camera = 0 | |
| off_camera = 0 | |
| total = 0 | |
| while True: | |
| # Read a single frame from the video | |
| for i in range(24*3): | |
| ret, frame = video_capture.read() | |
| if not ret: | |
| break | |
| # If there are no more frames, break out of the loop | |
| if not ret: | |
| break | |
| result, emotion = process_frame(frame) | |
| print(emotion) | |
| if result: | |
| if result == 'on_camera': | |
| on_camera += 1 | |
| elif result == 'off_camera': | |
| off_camera += 1 | |
| total += 1 | |
| if emotion != 0: | |
| # print(emotion[0]['emotion']) | |
| angry += emotion[0]['emotion']['angry'] | |
| disgust += emotion[0]['emotion']['disgust'] | |
| fear += emotion[0]['emotion']['fear'] | |
| happy += emotion[0]['emotion']['happy'] | |
| sad += emotion[0]['emotion']['sad'] | |
| surprise += emotion[0]['emotion']['surprise'] | |
| neutral += emotion[0]['emotion']['neutral'] | |
| try: | |
| # your processing code here | |
| gaze_percentage = on_camera / total * 100 | |
| except Exception as e: | |
| print(f"An error occurred while processing the video: {e}") | |
| gaze_percentage = f'no face detected Total = {total},on_camera = {on_camera},off_camera = {off_camera}' | |
| print(f'Total = {total},on_camera = {on_camera},off_camera = {off_camera}') | |
| # print(f'focus perfectage = {on_camera/total*100}') | |
| # Release the video capture object and close all windows | |
| video_capture.release() | |
| cv2.destroyAllWindows() | |
| end_time = time.time() | |
| print(f'Time taken: {end_time-start_time}') | |
| if os.path.exists("temp_video.mp4"): | |
| os.remove("temp_video.mp4") | |
| print(gaze_percentage) | |
| angry = angry / emotion_count | |
| disgust = disgust / emotion_count | |
| fear = fear / emotion_count | |
| happy = happy / emotion_count | |
| sad = sad / emotion_count | |
| surprise = surprise / emotion_count | |
| neutral = neutral / emotion_count | |
| emotion = { | |
| 'angry': angry, | |
| 'disgust': disgust, | |
| 'fear': fear, | |
| 'happy': happy, | |
| 'sad': sad, | |
| 'surprise': surprise, | |
| 'neutral': neutral | |
| }, | |
| final_result_dict = { | |
| "gaze_percentage" : gaze_percentage, | |
| "face_emotion" : emotion, | |
| "text_emotion" : text_emotion, | |
| "transcription" : transcription, | |
| "text_sentiment" : text_sentiment | |
| } | |
| # angry = 'total anger percentage' + str(angry) | |
| # disgust = 'total disgust percentage' + str(disgust) | |
| # fear = 'total fear percentage' + str(fear) | |
| # happy = 'total happy percentage' + str(happy) | |
| # sad = 'total sad percentage' + str(sad) | |
| # surprise = 'total surprise percentage' + str(surprise) | |
| # neutral = 'total neutral percentage' + str(neutral) | |
| print(f'total anger percentage = {angry}') | |
| print(f'total disgust percentage = {disgust}') | |
| print(f'total fear percentage = {fear}') | |
| print(f'total happy percentage = {happy}') | |
| print(f'total sad percentage = {sad}') | |
| print(f'total surprise percentage = {surprise}') | |
| print(f'total neutral percentage = {neutral}') | |
| final_result = "Gaze = "+str(gaze_percentage)+"\nFace Emotion = "+str(emotion)+"\nText Emotion = "+str(text_emotion)+"\nText transcription = "+str(transcription)+"\nText sentiment = "+str(text_sentiment) | |
| return final_result_dict | |
| demo = gr.Interface(fn=video_processing, | |
| inputs=["video", "text"], | |
| outputs="json") | |
| if __name__ == "__main__": | |
| demo.launch() |