pragnakalp commited on
Commit
7a325cf
·
1 Parent(s): cd07460

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +20 -178
app.py CHANGED
@@ -11,67 +11,10 @@ from gtts import gTTS
11
  import tempfile
12
  from pydub import AudioSegment
13
  from pydub.generators import Sine
14
- from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
15
- from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
16
- import dlib
17
- import cv2
18
- import imageio
19
- import ffmpeg
20
 
21
 
22
  block = gr.Blocks()
23
 
24
- def compute_aspect_preserved_bbox(bbox, increase_area, h, w):
25
- left, top, right, bot = bbox
26
- width = right - left
27
- height = bot - top
28
-
29
- width_increase = max(increase_area, ((1 + 2 * increase_area) * height - width) / (2 * width))
30
- height_increase = max(increase_area, ((1 + 2 * increase_area) * width - height) / (2 * height))
31
-
32
- left_t = int(left - width_increase * width)
33
- top_t = int(top - height_increase * height)
34
- right_t = int(right + width_increase * width)
35
- bot_t = int(bot + height_increase * height)
36
-
37
- left_oob = -min(0, left_t)
38
- right_oob = right - min(right_t, w)
39
- top_oob = -min(0, top_t)
40
- bot_oob = bot - min(bot_t, h)
41
-
42
- if max(left_oob, right_oob, top_oob, bot_oob) > 0:
43
- max_w = max(left_oob, right_oob)
44
- max_h = max(top_oob, bot_oob)
45
- if max_w > max_h:
46
- return left_t + max_w, top_t + max_w, right_t - max_w, bot_t - max_w
47
- else:
48
- return left_t + max_h, top_t + max_h, right_t - max_h, bot_t - max_h
49
-
50
- else:
51
- return (left_t, top_t, right_t, bot_t)
52
-
53
- def crop_src_image(src_img, detector=None):
54
- if detector is None:
55
- detector = dlib.get_frontal_face_detector()
56
- save_img='/content/image_pre.png'
57
- img = cv2.imread(src_img)
58
- faces = detector(img, 0)
59
- h, width, _ = img.shape
60
- if len(faces) > 0:
61
- bbox = [faces[0].left(), faces[0].top(),faces[0].right(), faces[0].bottom()]
62
- l = bbox[3]-bbox[1]
63
- bbox[1]= bbox[1]-l*0.1
64
- bbox[3]= bbox[3]-l*0.1
65
- bbox[1] = max(0,bbox[1])
66
- bbox[3] = min(h,bbox[3])
67
- bbox = compute_aspect_preserved_bbox(tuple(bbox), 0.5, img.shape[0], img.shape[1])
68
- img = img[bbox[1] :bbox[3] , bbox[0]:bbox[2]]
69
- img = cv2.resize(img, (256, 256))
70
- cv2.imwrite(save_img,img)
71
- else:
72
- img = cv2.resize(img,(256,256))
73
- cv2.imwrite(save_img, img)
74
-
75
  def pad_image(image):
76
  w, h = image.size
77
  if w == h:
@@ -87,6 +30,7 @@ def pad_image(image):
87
 
88
  def calculate(image_in, audio_in):
89
  waveform, sample_rate = torchaudio.load(audio_in)
 
90
  torchaudio.save("/content/audio.wav", waveform, sample_rate, encoding="PCM_S", bits_per_sample=16)
91
  image = Image.open(image_in)
92
  image = pad_image(image)
@@ -96,141 +40,39 @@ def calculate(image_in, audio_in):
96
  jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub("<S>"; "sil") | sub("<SIL>"; "sil") | sub("\\\(2\\\)"; "") | sub("\\\(3\\\)"; "") | sub("\\\(4\\\)"; "") | sub("\\\[SPEECH\\\]"; "SIL") | sub("\\\[NOISE\\\]"; "SIL")), phones: [.w[]|{ph: .t | sub("\\\+SPN\\\+"; "SIL") | sub("\\\+NSN\\\+"; "SIL"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)
97
  with open("test.json", "w") as f:
98
  f.write(jq_run.stdout.decode('utf-8').strip())
99
-
100
- os.system(f"cd /content/one-shot-talking-face && python3 -B test_script.py --img_path /content/results/restored_imgs/image_pre.png --audio_path /content/audio.wav --phoneme_path /content/test.json --save_dir /content/train")
101
  return "/content/train/image_audio.mp4"
102
-
103
-
104
- def merge_frames():
105
- import imageio
106
- import os
107
-
108
- path = '/content/video_results/restored_imgs'
109
- image_folder = os.fsencode(path)
110
- print(image_folder)
111
- filenames = []
112
-
113
- for file in os.listdir(image_folder):
114
- filename = os.fsdecode(file)
115
- if filename.endswith( ('.jpg', '.png', '.gif') ):
116
- filenames.append(filename)
117
-
118
- filenames.sort() # this iteration technique has no built in order, so sort the frames
119
- print(filenames)
120
- images = list(map(lambda filename: imageio.imread("/content/video_results/restored_imgs/"+filename), filenames))
121
-
122
-
123
- imageio.mimsave('/content/video_output.mp4', images, fps=25.0) # modify the frame duration as needed
124
-
125
-
126
-
127
- def audio_video():
128
-
129
- input_video = ffmpeg.input('/content/video_output.mp4')
130
-
131
- input_audio = ffmpeg.input('/content/audio.wav')
132
-
133
- ffmpeg.concat(input_video, input_audio, v=1, a=1).output('/content/final_output.mp4').run()
134
-
135
- return "/content/final_output.mp4"
136
-
137
- def one_shot_talking(image_in,audio_in):
138
-
139
-
140
- #Pre-processing of image
141
- crop_src_image(image_in)
142
-
143
- #Improve quality of input image
144
- os.system(f"python /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/image_pre.png -o /content/results --bg_upsampler realesrgan")
145
 
146
- image_in_one_shot='/content/results/restored_imgs/image_pre.png'
147
-
148
- #One Shot Talking Face algorithm
149
- calculate(image_in_one_shot,audio_in)
150
-
151
- #Video Quality Improvement
152
-
153
- #1. Extract the frames from the video file using PyVideoFramesExtractor
154
- os.system(f"python /content/PyVideoFramesExtractor/extract.py --video=/content/train/image_pre_audio.mp4")
155
-
156
- #2. Improve image quality using GFPGAN on each frames
157
- os.system(f"python /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/extracted_frames/image_pre_audio_frames -o /content/video_results --bg_upsampler realesrgan")
158
-
159
- #3. Merge all the frames to a one video using imageio
160
- merge_frames()
161
-
162
- return audio_video()
163
-
164
-
165
-
166
-
167
-
168
- def one_shot(image,input_text,gender):
169
- if gender == "Female":
170
- tts = gTTS(input_text)
171
- with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
172
- tts.write_to_fp(f)
173
- f.seek(0)
174
- sound = AudioSegment.from_file(f.name, format="mp3")
175
- sound.export("/content/audio.wav", format="wav")
176
- waveform, sample_rate = torchaudio.load("/content/audio.wav")
177
- audio_in="/content/audio.wav"
178
-
179
- return one_shot_talking(image,audio_in)
180
- elif gender == "Male":
181
-
182
- models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
183
- "Voicemod/fastspeech2-en-male1",
184
- arg_overrides={"vocoder": "hifigan", "fp16": False}
185
- )
186
-
187
- model = models[0].cuda()
188
- TTSHubInterface.update_cfg_with_data_cfg(cfg, task.data_cfg)
189
- generator = task.build_generator([model], cfg)
190
- # next(model.parameters()).device
191
-
192
- sample = TTSHubInterface.get_model_input(task, input_text)
193
- sample["net_input"]["src_tokens"] = sample["net_input"]["src_tokens"].cuda()
194
- sample["net_input"]["src_lengths"] = sample["net_input"]["src_lengths"].cuda()
195
- sample["speaker"] = sample["speaker"].cuda()
196
-
197
- wav, rate = TTSHubInterface.get_prediction(task, model, generator, sample)
198
- # soundfile.write("/content/audio_before.wav", wav, rate)
199
- soundfile.write("/content/audio_before.wav", wav.cpu().clone().numpy(), rate)
200
- cmd='ffmpeg -i /content/audio_before.wav -filter:a "atempo=0.7" -vn /content/audio.wav'
201
- os.system(cmd)
202
- return one_shot_talking(image,"/content/audio.wav")
203
-
204
-
205
-
206
-
207
- def generate_ocr(method,image,gender):
208
- return "Hello"
209
-
210
  def run():
211
  with block:
212
-
213
  with gr.Group():
214
  with gr.Box():
215
  with gr.Row().style(equal_height=True):
216
  image_in = gr.Image(show_label=False, type="filepath")
217
- # audio_in = gr.Audio(show_label=False, type='filepath')
218
- input_text=gr.Textbox(lines=3, value="Hello How are you?", label="Input Text")
219
  gender = gr.Radio(["Female","Male"],value="Female",label="Gender")
220
- video_out = gr.Textbox(label="output")
221
- # video_out = gr.Video(show_label=False)
222
  with gr.Row().style(equal_height=True):
223
  btn = gr.Button("Generate")
224
 
225
- btn.click(one_shot, inputs=[image_in, input_text,gender], outputs=[video_out])
226
- # block.queue()
 
227
  block.launch(server_name="0.0.0.0", server_port=7860)
228
 
229
  if __name__ == "__main__":
230
  run()
231
 
232
-
233
-
234
-
235
-
236
-
 
11
  import tempfile
12
  from pydub import AudioSegment
13
  from pydub.generators import Sine
 
 
 
 
 
 
14
 
15
 
16
  block = gr.Blocks()
17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  def pad_image(image):
19
  w, h = image.size
20
  if w == h:
 
30
 
31
  def calculate(image_in, audio_in):
32
  waveform, sample_rate = torchaudio.load(audio_in)
33
+ waveform = torch.mean(waveform, dim=0, keepdim=True)
34
  torchaudio.save("/content/audio.wav", waveform, sample_rate, encoding="PCM_S", bits_per_sample=16)
35
  image = Image.open(image_in)
36
  image = pad_image(image)
 
40
  jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub("<S>"; "sil") | sub("<SIL>"; "sil") | sub("\\\(2\\\)"; "") | sub("\\\(3\\\)"; "") | sub("\\\(4\\\)"; "") | sub("\\\[SPEECH\\\]"; "SIL") | sub("\\\[NOISE\\\]"; "SIL")), phones: [.w[]|{ph: .t | sub("\\\+SPN\\\+"; "SIL") | sub("\\\+NSN\\\+"; "SIL"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)
41
  with open("test.json", "w") as f:
42
  f.write(jq_run.stdout.decode('utf-8').strip())
43
+ # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
44
+ os.system(f"cd /content/one-shot-talking-face && python3 -B test_script.py --img_path /content/image.png --audio_path /content/audio.wav --phoneme_path /content/test.json --save_dir /content/train")
45
  return "/content/train/image_audio.mp4"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
 
47
+ def one_shot(image_in,input_text,gender):
48
+ if gender == "Female":
49
+ tts = gTTS(input_text)
50
+ with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
51
+ tts.write_to_fp(f)
52
+ f.seek(0)
53
+ sound = AudioSegment.from_file(f.name, format="mp3")
54
+ sound.export("/content/audio.wav", format="wav")
55
+ audio_in="/content/audio.wav"
56
+ return calculate(image_in,audio_in)
57
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  def run():
59
  with block:
60
+
61
  with gr.Group():
62
  with gr.Box():
63
  with gr.Row().style(equal_height=True):
64
  image_in = gr.Image(show_label=False, type="filepath")
65
+ input_text = gr.Textbox(show_label=False,label="Text")
 
66
  gender = gr.Radio(["Female","Male"],value="Female",label="Gender")
67
+ video_out = gr.Video(show_label=False)
 
68
  with gr.Row().style(equal_height=True):
69
  btn = gr.Button("Generate")
70
 
71
+
72
+ btn.click(one_shot, inputs=[image_in,input_text,gender], outputs=[video_out])
73
+ block.queue()
74
  block.launch(server_name="0.0.0.0", server_port=7860)
75
 
76
  if __name__ == "__main__":
77
  run()
78