Initial test

app.py

@@ -1,7 +1,325 @@
+import os
+import sys
 import gradio as gr
+import math
+import matplotlib.pyplot as plt
+
+import requests
+import fileinput
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+import gradio as gr
+import json
+import math
+import requests
+
+
+basedir = "/content/drive/MyDrive/FYP/Code/VideoCrafter"
+# if you change the variables here, remember to change the "name" in .sh file
+vidOut = "results/10videos"
+uvqOut = "results/modified_prompts_eval"
+evalOut = "evaluation_results"
+num_of_vid = 3
+vid_length = 2
+uvq_threshold = 3.8
+fps = 24
+
+
+# Generate the scores in csv files
+def genScore():
+    for i in range(1, num_of_vid+1):
+        fileindex = f"{i:04d}"
+        os.system(
+            f'python3 ./uvq/uvq_main.py --input_files="{fileindex},2,{basedir}/{vidOut}/{fileindex}.mp4" --output_dir {uvqOut} --model_dir ./uvq/models'
+        )
+
+
+def getScore(filename):
+    # MOS_score defines the output of the uvq score
+    lines = str(filename).split('\n')
+    last_line = lines[-1]
+    MOS_score = last_line.split(',')[-1]
+    MOS_score = MOS_score[:-2]
+
+    return MOS_score
+
+# MOS_score defines the Mean Opinion Score of prediction, if the video's MOS exceeds the threshold then we directly use this video
+
+
+def chooseBestVideo():
+    MOS_score_high = 0
+    preferred_output = ""
+    chosen_idx = 0
+
+    for i in range(1, num_of_vid+1):
+        '''We loop thru this current processed video'''
+        filedir = f"{i:04d}"
+        filename = f"{i:04d}_uvq.csv"
+        with open(os.path.join(basedir, uvqOut, filedir, filename), 'r') as file:
+            MOS = file.read().strip()
+
+        MOS_score = getScore(MOS)
+        print("Video Index:", f"{i:04d}", "Score:", MOS_score)
+
+        # if the MOS_score is higher than the previous video, we choose this video as our preferred video output
+        if float(MOS_score) > float(MOS_score_high) or float(MOS_score) > uvq_threshold:
+            MOS_score_high = MOS_score
+            preferred_output = filename
+            chosen_idx = i
+
+        if float(MOS_score) > uvq_threshold:
+            break
+    return chosen_idx
+    # print(MOS_score_high)
+    # print(preferred_output)
+
+
+def extract_scores_from_json(json_path):
+    with open(json_path) as file:
+        data = json.load(file)
+
+    for key, value in data.items():
+        if isinstance(value, list) and len(value) > 1 and isinstance(value[0], float):
+            motion_score = value[0]
+
+    return motion_score
+
+
+def VBench_eval(vid_filename):
+    # vid_filename: video filename without .mp4
+    os.system(
+        f'python VBench/evaluate.py --dimension "motion_smoothness"  --videos_path {os.path.join(basedir, vidOut, vid_filename)}.mp4 --custom_input --output_filename {vid_filename}'
+    )
+    eval_file_path = os.path.join(
+        basedir, evalOut, f"{vid_filename}_eval_results.json")
+    motion_score = extract_scores_from_json(eval_file_path)
+
+    return motion_score
+
+
+def interpolation(chosen_idx, fps):
+    vid_filename = f"{chosen_idx:04d}.mp4"
+    os.chdir("/content/drive/MyDrive/FYP/Code/VideoCrafter/ECCV2022-RIFE")
+    os.system(
+        f'python3 inference_video.py --exp=2 --video={os.path.join(basedir, vidOut, vid_filename)} --fps {fps}'
+    )
+    os.chdir("/content/drive/MyDrive/FYP/Code/VideoCrafter")
+    out_name = f"{chosen_idx:04d}_4X_{fps}fps.mp4"
+    return out_name
+
+# call the GPT API here
+
+
+def call_gpt_api(prompt, isSentence=False):
+    api_key = "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"
+
+    response = requests.post(
+        'https://api.openai.com/v1/chat/completions',
+        headers={
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {api_key}'
+        },
+        json={
+            'messages': [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': prompt}],
+            'model': 'gpt-3.5-turbo',
+            # 'prompt': prompt,
+            'temperature': 0.4,
+            'max_tokens': 200
+        })
+    response_json = response.json()
+    choices = response_json['choices']
+    contents = [choice['message']['content'] for choice in choices]
+    contents = [
+        sentence for sublist in contents for sentence in sublist.split('\n')]
+    # Remove the leading number and dot from each sentence
+    sentences = [content.lstrip('1234567890.- ') for content in contents]
+    if len(sentences) > 2 and isSentence:
+        sentences = sentences[1:]
+    return sentences
+
+
+# Initialize Firebase Admin SDK
+cred = credentials.Certificate(
+    f"{basedir}/final-year-project-443dd-df6f48af0796.json")
+firebase_admin.initialize_app(cred)
+# Initialize Firestore client
+db = firestore.client()
+
+
+def retrieve_user_feedback():
+    # Retrieve user feedback from Firestore
+    feedback_collection = db.collection("user_feedbacks")
+    feedback_docs = feedback_collection.get()
+
+    feedback_text = []
+    experience = []
+    for doc in feedback_docs:
+        data = doc.to_dict()
+        feedback_text.append(data.get('feedback_text', None))
+        experience.append(data.get('experience', None))
+
+    return feedback_text, experience
+
+
+feedback_text, experience = retrieve_user_feedback()
+# print("Feedback Text:", feedback_text)
+# print("Experience:", experience)
+
+
+def store_user_feedback(feedback_text, experience):
+    # Get a reference to the Firestore collection
+    feedback_collection = db.collection("user_feedbacks")
+
+    # Create a new document with feedback_text and experience fields
+    feedback_collection.add({
+        'feedback_text': feedback_text,
+        'experience': experience
+    })
+    return
+
+
+t2v_examples = [
+    ['A tiger walks in the forest, photorealistic, 4k, high definition'],
+    ['an elephant is walking under the sea, 4K, high definition'],
+    ['an astronaut riding a horse in outer space'],
+    ['a monkey is playing a piano'],
+    ['A fire is burning on a candle'],
+    ['a horse is drinking in the river'],
+    ['Robot dancing in times square'],
+]
+
+
+def generate_output(input_text, output_video_1, fps, examples):
+    def generate_output_fn(input_text, output_video_1, fps, examples):
+        if input_text == "":
+            return input_text, output_video_1, examples
+        output = call_gpt_api(
+            prompt=f"Generate 2 similar prompts and add some reasonable words to the given prompt and not change the meaning, each within 30 words: {input_text}", isSentence=True)
+        output.append(input_text)
+        with open(f"{basedir}/prompts/test_prompts.txt", 'w') as file:
+            for i, sentence in enumerate(output):
+                if i < len(output) - 1:
+                    file.write(sentence + '\n')
+                else:
+                    file.write(sentence)
+        os.system(
+            f'sh {os.path.join(basedir, "scripts", "run_text2video.sh")}')
+        # Connect the video output and return the video corresponding link
+        genScore()
+        chosen_idx = chooseBestVideo()
+        chosen_vid_path = interpolation(chosen_idx, fps)
+        chosen_vid_path = f"{basedir}/{vidOut}/{chosen_vid_path}"
+        # chosen_vid_path = "/content/drive/MyDrive/FYP/Code/VideoCrafter/results/cat/0002_4X_16fps.mp4"
+        output_video_1 = gr.Video(
+            value=chosen_vid_path, show_download_button=True)
+
+        examples_list = call_gpt_api(
+            prompt=f"Generate 5 similar prompts that makes a storyline coming after the given input, each within 10 words: {input_text}")
+        examples = []
+        for prompt in examples_list:
+            examples.append([prompt])
+        input_text = ""
+
+        return input_text, output_video_1, examples
+
+    return generate_output_fn(input_text, output_video_1, fps, examples)
+
+
+def t2v_demo(result_dir='./tmp/'):
+    with gr.Blocks() as videocrafter_iface:
+        gr.Markdown("<div align='center'> <h2> VideoCraftXtend: AI-Enhanced Text-to-Video Generation with Extended Length and Enhanced Motion Smoothness </span> </h2> </div>")
+
+        # Initialize values for video length and fps
+        video_len_value = 5.0
+
+        def update_fps(video_len, fps):
+            fps_value = 80 / video_len
+            return f"<div justify-content: 'center'; text-align='center'> <h6> FPS (frames per second) : {int(fps_value)} </span> </h6> </div>"
+
+        def load_example(example_id):
+            return example_id[0]
+
+        def update_feedback(value, text):
+            labels = ['Positive', 'Neutral', 'Negative']
+            colors = ['#66c2a5', '#fc8d62', '#8da0cb']
+            if value != '':
+                store_user_feedback(value, text)
+                user_satisfaction.append(value)
+                value = ''
+            if text != '':
+                user_feedback.append(text)
+                text = ''
+            user_feedback, user_satisfaction = retrieve_user_feedback()
+            sizes = [user_satisfaction.count('Positive'), user_satisfaction.count(
+                'Neutral'), user_satisfaction.count('Negative')]
+            plt.pie(sizes, labels=labels, autopct='%1.1f%%',
+                    startangle=140, colors=colors)
+            plt.axis('equal')
+            return plt
+
+        with gr.Tab(label="Text2Video"):
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        input_text = gr.Text(
+                            placeholder=t2v_examples[2], label='Please input your prompt here.')
+                        with gr.Row():
+                            examples = gr.Dataset(samples=t2v_examples, components=[
+                                                  input_text], label='Sample prompts that can be used to form a storyline.')
+                        with gr.Column():
+                            gr.Markdown(
+                                "<div align='center'> <h4> Modify video length and the corresponding fps will be shown on the right. </span> </h4> </div>")
+                            with gr.Row():
+                                video_len = gr.Slider(minimum=4.0, maximum=10.0, step=1, label='Video Length',
+                                                      value=video_len_value, elem_id="video_len", interactive=True)
+                                fps = gr.Markdown(
+                                    elem_id="fps", value=f"<div> <h6> FPS (frames per second) : 16</span> </h6> </div>")
+                        send_btn = gr.Button("Send")
+                    with gr.Column():
+                        with gr.Tab(label='Result'):
+                            with gr.Row():
+                                output_video_1 = gr.Video(
+                                    value="/content/drive/MyDrive/FYP/Code/VideoCrafter/results/10videos/0009.mp4", show_download_button=True)
+
+            video_len.change(update_fps, inputs=[video_len, fps], outputs=fps)
+            # fps.change(update_video_len_slider, inputs = fps, outputs = video_len)
+
+            examples.click(load_example, inputs=[
+                           examples], outputs=[input_text])
+            send_btn.click(
+                fn=generate_output,
+                inputs=[input_text, output_video_1, fps, examples],
+                outputs=[input_text, output_video_1, examples],
+            )
+
+        with gr.Tab(label="Feedback"):
+            with gr.Column():
+                with gr.Column():
+                    with gr.Row():
+                        feedback_value = gr.Radio(
+                            ['Positive', 'Neutral', 'Negative'], label="How is your experience?")
+                        feedback_text = gr.Textbox(
+                            placeholder="Enter feedback here", label="Feedback Text")
+                    with gr.Row():
+                        cancel_btn = gr.Button("Clear")
+                        submit_btn = gr.Button("Submit")
+                with gr.Row():
+                    pie_chart = gr.Plot(value=update_feedback(
+                        '', ''), label="Feedback Pie Chart")
+                    with gr.Column():
+                        gr.Markdown(
+                            "<div align='center'> <h4> Feedbacks from users: </span> </h4> </div>")
+                        feedback_text_display = [gr.Markdown(
+                            feedback, label="User Feedback") for feedback in retrieve_user_feedback()[0]]
+                submit_btn.click(fn=update_feedback, inputs=[
+                                 feedback_value, feedback_text], outputs=[pie_chart])
+
+    return videocrafter_iface
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+if __name__ == "__main__":
+    result_dir = os.path.join('./', 'results')
+    t2v_iface = t2v_demo(result_dir)
+    t2v_iface.queue(max_size=10)
+    t2v_iface.launch(debug=True)