Commit
·
7a325cf
1
Parent(s):
cd07460
Update app.py
Browse files
app.py
CHANGED
@@ -11,67 +11,10 @@ from gtts import gTTS
|
|
11 |
import tempfile
|
12 |
from pydub import AudioSegment
|
13 |
from pydub.generators import Sine
|
14 |
-
from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
|
15 |
-
from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
|
16 |
-
import dlib
|
17 |
-
import cv2
|
18 |
-
import imageio
|
19 |
-
import ffmpeg
|
20 |
|
21 |
|
22 |
block = gr.Blocks()
|
23 |
|
24 |
-
def compute_aspect_preserved_bbox(bbox, increase_area, h, w):
|
25 |
-
left, top, right, bot = bbox
|
26 |
-
width = right - left
|
27 |
-
height = bot - top
|
28 |
-
|
29 |
-
width_increase = max(increase_area, ((1 + 2 * increase_area) * height - width) / (2 * width))
|
30 |
-
height_increase = max(increase_area, ((1 + 2 * increase_area) * width - height) / (2 * height))
|
31 |
-
|
32 |
-
left_t = int(left - width_increase * width)
|
33 |
-
top_t = int(top - height_increase * height)
|
34 |
-
right_t = int(right + width_increase * width)
|
35 |
-
bot_t = int(bot + height_increase * height)
|
36 |
-
|
37 |
-
left_oob = -min(0, left_t)
|
38 |
-
right_oob = right - min(right_t, w)
|
39 |
-
top_oob = -min(0, top_t)
|
40 |
-
bot_oob = bot - min(bot_t, h)
|
41 |
-
|
42 |
-
if max(left_oob, right_oob, top_oob, bot_oob) > 0:
|
43 |
-
max_w = max(left_oob, right_oob)
|
44 |
-
max_h = max(top_oob, bot_oob)
|
45 |
-
if max_w > max_h:
|
46 |
-
return left_t + max_w, top_t + max_w, right_t - max_w, bot_t - max_w
|
47 |
-
else:
|
48 |
-
return left_t + max_h, top_t + max_h, right_t - max_h, bot_t - max_h
|
49 |
-
|
50 |
-
else:
|
51 |
-
return (left_t, top_t, right_t, bot_t)
|
52 |
-
|
53 |
-
def crop_src_image(src_img, detector=None):
|
54 |
-
if detector is None:
|
55 |
-
detector = dlib.get_frontal_face_detector()
|
56 |
-
save_img='/content/image_pre.png'
|
57 |
-
img = cv2.imread(src_img)
|
58 |
-
faces = detector(img, 0)
|
59 |
-
h, width, _ = img.shape
|
60 |
-
if len(faces) > 0:
|
61 |
-
bbox = [faces[0].left(), faces[0].top(),faces[0].right(), faces[0].bottom()]
|
62 |
-
l = bbox[3]-bbox[1]
|
63 |
-
bbox[1]= bbox[1]-l*0.1
|
64 |
-
bbox[3]= bbox[3]-l*0.1
|
65 |
-
bbox[1] = max(0,bbox[1])
|
66 |
-
bbox[3] = min(h,bbox[3])
|
67 |
-
bbox = compute_aspect_preserved_bbox(tuple(bbox), 0.5, img.shape[0], img.shape[1])
|
68 |
-
img = img[bbox[1] :bbox[3] , bbox[0]:bbox[2]]
|
69 |
-
img = cv2.resize(img, (256, 256))
|
70 |
-
cv2.imwrite(save_img,img)
|
71 |
-
else:
|
72 |
-
img = cv2.resize(img,(256,256))
|
73 |
-
cv2.imwrite(save_img, img)
|
74 |
-
|
75 |
def pad_image(image):
|
76 |
w, h = image.size
|
77 |
if w == h:
|
@@ -87,6 +30,7 @@ def pad_image(image):
|
|
87 |
|
88 |
def calculate(image_in, audio_in):
|
89 |
waveform, sample_rate = torchaudio.load(audio_in)
|
|
|
90 |
torchaudio.save("/content/audio.wav", waveform, sample_rate, encoding="PCM_S", bits_per_sample=16)
|
91 |
image = Image.open(image_in)
|
92 |
image = pad_image(image)
|
@@ -96,141 +40,39 @@ def calculate(image_in, audio_in):
|
|
96 |
jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub("<S>"; "sil") | sub("<SIL>"; "sil") | sub("\\\(2\\\)"; "") | sub("\\\(3\\\)"; "") | sub("\\\(4\\\)"; "") | sub("\\\[SPEECH\\\]"; "SIL") | sub("\\\[NOISE\\\]"; "SIL")), phones: [.w[]|{ph: .t | sub("\\\+SPN\\\+"; "SIL") | sub("\\\+NSN\\\+"; "SIL"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)
|
97 |
with open("test.json", "w") as f:
|
98 |
f.write(jq_run.stdout.decode('utf-8').strip())
|
99 |
-
|
100 |
-
os.system(f"cd /content/one-shot-talking-face && python3 -B test_script.py --img_path /content/
|
101 |
return "/content/train/image_audio.mp4"
|
102 |
-
|
103 |
-
|
104 |
-
def merge_frames():
|
105 |
-
import imageio
|
106 |
-
import os
|
107 |
-
|
108 |
-
path = '/content/video_results/restored_imgs'
|
109 |
-
image_folder = os.fsencode(path)
|
110 |
-
print(image_folder)
|
111 |
-
filenames = []
|
112 |
-
|
113 |
-
for file in os.listdir(image_folder):
|
114 |
-
filename = os.fsdecode(file)
|
115 |
-
if filename.endswith( ('.jpg', '.png', '.gif') ):
|
116 |
-
filenames.append(filename)
|
117 |
-
|
118 |
-
filenames.sort() # this iteration technique has no built in order, so sort the frames
|
119 |
-
print(filenames)
|
120 |
-
images = list(map(lambda filename: imageio.imread("/content/video_results/restored_imgs/"+filename), filenames))
|
121 |
-
|
122 |
-
|
123 |
-
imageio.mimsave('/content/video_output.mp4', images, fps=25.0) # modify the frame duration as needed
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
def audio_video():
|
128 |
-
|
129 |
-
input_video = ffmpeg.input('/content/video_output.mp4')
|
130 |
-
|
131 |
-
input_audio = ffmpeg.input('/content/audio.wav')
|
132 |
-
|
133 |
-
ffmpeg.concat(input_video, input_audio, v=1, a=1).output('/content/final_output.mp4').run()
|
134 |
-
|
135 |
-
return "/content/final_output.mp4"
|
136 |
-
|
137 |
-
def one_shot_talking(image_in,audio_in):
|
138 |
-
|
139 |
-
|
140 |
-
#Pre-processing of image
|
141 |
-
crop_src_image(image_in)
|
142 |
-
|
143 |
-
#Improve quality of input image
|
144 |
-
os.system(f"python /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/image_pre.png -o /content/results --bg_upsampler realesrgan")
|
145 |
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
os.system(f"python /content/GFPGAN/inference_gfpgan.py --upscale 2 -i /content/extracted_frames/image_pre_audio_frames -o /content/video_results --bg_upsampler realesrgan")
|
158 |
-
|
159 |
-
#3. Merge all the frames to a one video using imageio
|
160 |
-
merge_frames()
|
161 |
-
|
162 |
-
return audio_video()
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
def one_shot(image,input_text,gender):
|
169 |
-
if gender == "Female":
|
170 |
-
tts = gTTS(input_text)
|
171 |
-
with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
|
172 |
-
tts.write_to_fp(f)
|
173 |
-
f.seek(0)
|
174 |
-
sound = AudioSegment.from_file(f.name, format="mp3")
|
175 |
-
sound.export("/content/audio.wav", format="wav")
|
176 |
-
waveform, sample_rate = torchaudio.load("/content/audio.wav")
|
177 |
-
audio_in="/content/audio.wav"
|
178 |
-
|
179 |
-
return one_shot_talking(image,audio_in)
|
180 |
-
elif gender == "Male":
|
181 |
-
|
182 |
-
models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
|
183 |
-
"Voicemod/fastspeech2-en-male1",
|
184 |
-
arg_overrides={"vocoder": "hifigan", "fp16": False}
|
185 |
-
)
|
186 |
-
|
187 |
-
model = models[0].cuda()
|
188 |
-
TTSHubInterface.update_cfg_with_data_cfg(cfg, task.data_cfg)
|
189 |
-
generator = task.build_generator([model], cfg)
|
190 |
-
# next(model.parameters()).device
|
191 |
-
|
192 |
-
sample = TTSHubInterface.get_model_input(task, input_text)
|
193 |
-
sample["net_input"]["src_tokens"] = sample["net_input"]["src_tokens"].cuda()
|
194 |
-
sample["net_input"]["src_lengths"] = sample["net_input"]["src_lengths"].cuda()
|
195 |
-
sample["speaker"] = sample["speaker"].cuda()
|
196 |
-
|
197 |
-
wav, rate = TTSHubInterface.get_prediction(task, model, generator, sample)
|
198 |
-
# soundfile.write("/content/audio_before.wav", wav, rate)
|
199 |
-
soundfile.write("/content/audio_before.wav", wav.cpu().clone().numpy(), rate)
|
200 |
-
cmd='ffmpeg -i /content/audio_before.wav -filter:a "atempo=0.7" -vn /content/audio.wav'
|
201 |
-
os.system(cmd)
|
202 |
-
return one_shot_talking(image,"/content/audio.wav")
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
|
207 |
-
def generate_ocr(method,image,gender):
|
208 |
-
return "Hello"
|
209 |
-
|
210 |
def run():
|
211 |
with block:
|
212 |
-
|
213 |
with gr.Group():
|
214 |
with gr.Box():
|
215 |
with gr.Row().style(equal_height=True):
|
216 |
image_in = gr.Image(show_label=False, type="filepath")
|
217 |
-
|
218 |
-
input_text=gr.Textbox(lines=3, value="Hello How are you?", label="Input Text")
|
219 |
gender = gr.Radio(["Female","Male"],value="Female",label="Gender")
|
220 |
-
video_out = gr.
|
221 |
-
# video_out = gr.Video(show_label=False)
|
222 |
with gr.Row().style(equal_height=True):
|
223 |
btn = gr.Button("Generate")
|
224 |
|
225 |
-
|
226 |
-
|
|
|
227 |
block.launch(server_name="0.0.0.0", server_port=7860)
|
228 |
|
229 |
if __name__ == "__main__":
|
230 |
run()
|
231 |
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
|
|
11 |
import tempfile
|
12 |
from pydub import AudioSegment
|
13 |
from pydub.generators import Sine
|
|
|
|
|
|
|
|
|
|
|
|
|
14 |
|
15 |
|
16 |
block = gr.Blocks()
|
17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
def pad_image(image):
|
19 |
w, h = image.size
|
20 |
if w == h:
|
|
|
30 |
|
31 |
def calculate(image_in, audio_in):
|
32 |
waveform, sample_rate = torchaudio.load(audio_in)
|
33 |
+
waveform = torch.mean(waveform, dim=0, keepdim=True)
|
34 |
torchaudio.save("/content/audio.wav", waveform, sample_rate, encoding="PCM_S", bits_per_sample=16)
|
35 |
image = Image.open(image_in)
|
36 |
image = pad_image(image)
|
|
|
40 |
jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub("<S>"; "sil") | sub("<SIL>"; "sil") | sub("\\\(2\\\)"; "") | sub("\\\(3\\\)"; "") | sub("\\\(4\\\)"; "") | sub("\\\[SPEECH\\\]"; "SIL") | sub("\\\[NOISE\\\]"; "SIL")), phones: [.w[]|{ph: .t | sub("\\\+SPN\\\+"; "SIL") | sub("\\\+NSN\\\+"; "SIL"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)
|
41 |
with open("test.json", "w") as f:
|
42 |
f.write(jq_run.stdout.decode('utf-8').strip())
|
43 |
+
# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
44 |
+
os.system(f"cd /content/one-shot-talking-face && python3 -B test_script.py --img_path /content/image.png --audio_path /content/audio.wav --phoneme_path /content/test.json --save_dir /content/train")
|
45 |
return "/content/train/image_audio.mp4"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
46 |
|
47 |
+
def one_shot(image_in,input_text,gender):
|
48 |
+
if gender == "Female":
|
49 |
+
tts = gTTS(input_text)
|
50 |
+
with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f:
|
51 |
+
tts.write_to_fp(f)
|
52 |
+
f.seek(0)
|
53 |
+
sound = AudioSegment.from_file(f.name, format="mp3")
|
54 |
+
sound.export("/content/audio.wav", format="wav")
|
55 |
+
audio_in="/content/audio.wav"
|
56 |
+
return calculate(image_in,audio_in)
|
57 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
def run():
|
59 |
with block:
|
60 |
+
|
61 |
with gr.Group():
|
62 |
with gr.Box():
|
63 |
with gr.Row().style(equal_height=True):
|
64 |
image_in = gr.Image(show_label=False, type="filepath")
|
65 |
+
input_text = gr.Textbox(show_label=False,label="Text")
|
|
|
66 |
gender = gr.Radio(["Female","Male"],value="Female",label="Gender")
|
67 |
+
video_out = gr.Video(show_label=False)
|
|
|
68 |
with gr.Row().style(equal_height=True):
|
69 |
btn = gr.Button("Generate")
|
70 |
|
71 |
+
|
72 |
+
btn.click(one_shot, inputs=[image_in,input_text,gender], outputs=[video_out])
|
73 |
+
block.queue()
|
74 |
block.launch(server_name="0.0.0.0", server_port=7860)
|
75 |
|
76 |
if __name__ == "__main__":
|
77 |
run()
|
78 |
|
|
|
|
|
|
|
|
|
|