Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -91,16 +91,6 @@ def get_args():
|
|
91 |
return args
|
92 |
|
93 |
|
94 |
-
args = get_args()
|
95 |
-
ensure_dirname(args.output_dir)
|
96 |
-
|
97 |
-
|
98 |
-
color_list = []
|
99 |
-
for i in range(20):
|
100 |
-
color = np.concatenate([np.random.random(4)*255], axis=0)
|
101 |
-
color_list.append(color)
|
102 |
-
|
103 |
-
|
104 |
def interpolate_trajectory(points, n_points):
|
105 |
x = [point[0] for point in points]
|
106 |
y = [point[1] for point in points]
|
@@ -536,264 +526,275 @@ class Drag:
|
|
536 |
return val_save_dir
|
537 |
|
538 |
|
539 |
-
|
540 |
-
gr.Markdown("""<h1 align="center">Framer: Interactive Frame Interpolation</h1><br>""")
|
541 |
|
542 |
-
|
543 |
-
|
544 |
-
|
545 |
-
|
546 |
-
gr.Image(label="Framer: Interactive Frame Interpolation", value="assets/demos.gif", height=432, width=768)
|
547 |
-
|
548 |
-
gr.Markdown("""## Usage: <br>
|
549 |
-
1. Upload images<br>
|
550 |
-
  1.1 Upload the start image via the "Upload Start Image" button.<br>
|
551 |
-
  1.2. Upload the end image via the "Upload End Image" button.<br>
|
552 |
-
2. (Optional) Draw some drags.<br>
|
553 |
-
  2.1. Click "Add Drag Trajectory" to add the motion trajectory.<br>
|
554 |
-
  2.2. You can click several points on either start or end image to forms a path.<br>
|
555 |
-
  2.3. Click "Delete last drag" to delete the whole lastest path.<br>
|
556 |
-
  2.4. Click "Delete last step" to delete the lastest clicked control point.<br>
|
557 |
-
3. Interpolate the images (according the path) with a click on "Run" button. <br>""")
|
558 |
|
559 |
-
|
560 |
-
|
561 |
-
|
562 |
-
|
563 |
-
tracking_points = gr.State([])
|
564 |
|
565 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
566 |
first_frame_path = gr.State()
|
567 |
last_frame_path = gr.State()
|
568 |
tracking_points = gr.State([])
|
569 |
-
|
570 |
-
|
571 |
-
|
572 |
-
|
573 |
-
|
574 |
-
|
575 |
-
|
576 |
-
|
577 |
-
|
578 |
-
|
579 |
-
|
580 |
-
|
581 |
-
|
582 |
-
|
583 |
-
|
584 |
-
|
585 |
-
|
586 |
-
|
587 |
-
|
588 |
-
|
589 |
-
|
590 |
-
|
591 |
-
|
592 |
-
|
593 |
-
|
594 |
-
|
595 |
-
|
596 |
-
|
597 |
-
|
598 |
-
|
599 |
-
|
600 |
-
|
601 |
-
|
602 |
-
|
603 |
-
|
604 |
-
|
605 |
-
|
606 |
-
|
607 |
-
|
608 |
-
|
609 |
-
|
610 |
-
|
611 |
-
|
612 |
-
|
613 |
-
|
614 |
-
|
615 |
-
|
616 |
-
|
617 |
-
|
618 |
-
|
619 |
-
|
620 |
-
|
621 |
-
|
622 |
-
|
623 |
-
|
624 |
-
|
625 |
-
|
626 |
-
|
627 |
-
|
628 |
-
|
629 |
-
|
630 |
-
|
631 |
-
|
632 |
-
|
633 |
-
|
634 |
-
|
635 |
-
|
636 |
-
|
637 |
-
|
638 |
-
|
639 |
-
|
640 |
-
|
641 |
-
|
642 |
-
|
643 |
-
|
644 |
-
|
645 |
-
|
646 |
-
|
647 |
-
|
648 |
-
|
649 |
-
|
650 |
-
|
651 |
-
|
652 |
-
|
653 |
-
|
654 |
-
|
655 |
-
|
656 |
-
|
657 |
-
|
658 |
-
|
659 |
-
|
660 |
-
|
661 |
-
|
662 |
-
|
663 |
-
|
664 |
-
|
665 |
-
|
666 |
-
|
667 |
-
|
668 |
-
|
669 |
-
|
670 |
-
|
671 |
-
|
672 |
-
|
673 |
-
|
674 |
-
|
675 |
-
|
676 |
-
|
677 |
-
|
678 |
-
|
679 |
-
|
680 |
-
|
681 |
-
|
682 |
-
|
683 |
-
|
684 |
-
|
685 |
-
|
686 |
-
|
687 |
-
|
688 |
-
|
689 |
-
|
690 |
-
|
691 |
-
|
692 |
-
|
693 |
-
|
694 |
-
|
695 |
-
|
696 |
-
|
697 |
-
|
698 |
-
|
699 |
-
|
700 |
-
|
701 |
-
|
702 |
-
|
703 |
-
|
704 |
-
|
705 |
-
|
706 |
-
|
707 |
-
|
708 |
-
|
709 |
-
|
710 |
-
|
711 |
-
|
712 |
-
|
713 |
-
|
714 |
-
|
715 |
-
|
716 |
-
|
717 |
-
|
718 |
-
|
719 |
-
|
720 |
-
|
721 |
-
|
722 |
-
|
723 |
-
|
724 |
-
|
725 |
-
|
726 |
-
|
727 |
-
|
728 |
-
|
729 |
-
|
730 |
-
|
731 |
-
|
732 |
-
|
733 |
-
|
734 |
-
|
735 |
-
|
736 |
-
|
737 |
-
|
738 |
-
|
739 |
-
|
740 |
-
|
741 |
-
|
742 |
-
|
743 |
-
|
744 |
-
|
745 |
-
|
746 |
-
|
747 |
-
|
748 |
-
|
749 |
-
|
750 |
-
|
751 |
-
|
752 |
-
|
753 |
-
|
754 |
-
|
755 |
-
|
756 |
-
|
757 |
-
|
758 |
-
|
759 |
-
|
760 |
-
|
761 |
-
|
762 |
-
|
763 |
-
|
764 |
-
|
765 |
-
)
|
766 |
-
|
767 |
-
|
768 |
-
|
769 |
-
|
770 |
-
|
771 |
-
|
772 |
-
|
773 |
-
|
774 |
-
|
775 |
-
|
776 |
-
|
777 |
-
|
778 |
-
|
779 |
-
|
780 |
-
|
781 |
-
|
782 |
-
|
783 |
-
|
784 |
-
|
785 |
-
|
786 |
-
|
787 |
-
|
788 |
-
|
789 |
-
|
790 |
-
|
791 |
-
|
792 |
-
|
793 |
-
|
794 |
-
|
795 |
-
|
796 |
-
|
797 |
-
|
798 |
-
|
799 |
-
|
|
|
|
|
|
|
|
|
|
|
|
91 |
return args
|
92 |
|
93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
94 |
def interpolate_trajectory(points, n_points):
|
95 |
x = [point[0] for point in points]
|
96 |
y = [point[1] for point in points]
|
|
|
526 |
return val_save_dir
|
527 |
|
528 |
|
529 |
+
if __name__ == "__main__":
|
|
|
530 |
|
531 |
+
args = get_args()
|
532 |
+
ensure_dirname(args.output_dir)
|
533 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
534 |
|
535 |
+
color_list = []
|
536 |
+
for i in range(20):
|
537 |
+
color = np.concatenate([np.random.random(4)*255], axis=0)
|
538 |
+
color_list.append(color)
|
|
|
539 |
|
540 |
+
with gr.Blocks() as demo:
|
541 |
+
gr.Markdown("""<h1 align="center">Framer: Interactive Frame Interpolation</h1><br>""")
|
542 |
+
|
543 |
+
gr.Markdown("""Gradio Demo for <a href='https://arxiv.org/abs/2410.18978'><b>Framer: Interactive Frame Interpolation</b></a>.<br>
|
544 |
+
Github Repo can be found at https://github.com/aim-uofa/Framer<br>
|
545 |
+
The template is inspired by DragAnything.""")
|
546 |
+
|
547 |
+
gr.Image(label="Framer: Interactive Frame Interpolation", value="assets/demos.gif", height=432, width=768)
|
548 |
+
|
549 |
+
gr.Markdown("""## Usage: <br>
|
550 |
+
1. Upload images<br>
|
551 |
+
  1.1 Upload the start image via the "Upload Start Image" button.<br>
|
552 |
+
  1.2. Upload the end image via the "Upload End Image" button.<br>
|
553 |
+
2. (Optional) Draw some drags.<br>
|
554 |
+
  2.1. Click "Add Drag Trajectory" to add the motion trajectory.<br>
|
555 |
+
  2.2. You can click several points on either start or end image to forms a path.<br>
|
556 |
+
  2.3. Click "Delete last drag" to delete the whole lastest path.<br>
|
557 |
+
  2.4. Click "Delete last step" to delete the lastest clicked control point.<br>
|
558 |
+
3. Interpolate the images (according the path) with a click on "Run" button. <br>""")
|
559 |
+
|
560 |
+
# device, args, height, width, model_length
|
561 |
+
Framer = Drag("cuda", args, 320, 512, 14)
|
562 |
first_frame_path = gr.State()
|
563 |
last_frame_path = gr.State()
|
564 |
tracking_points = gr.State([])
|
565 |
+
|
566 |
+
def reset_states(first_frame_path, last_frame_path, tracking_points):
|
567 |
+
first_frame_path = gr.State()
|
568 |
+
last_frame_path = gr.State()
|
569 |
+
tracking_points = gr.State([])
|
570 |
+
|
571 |
+
return first_frame_path, last_frame_path, tracking_points
|
572 |
+
|
573 |
+
|
574 |
+
def preprocess_image(image):
|
575 |
+
|
576 |
+
image_pil = image2pil(image.name)
|
577 |
+
|
578 |
+
raw_w, raw_h = image_pil.size
|
579 |
+
# resize_ratio = max(512 / raw_w, 320 / raw_h)
|
580 |
+
# image_pil = image_pil.resize((int(raw_w * resize_ratio), int(raw_h * resize_ratio)), Image.BILINEAR)
|
581 |
+
# image_pil = transforms.CenterCrop((320, 512))(image_pil.convert('RGB'))
|
582 |
+
image_pil = image_pil.resize((512, 320), Image.BILINEAR)
|
583 |
+
|
584 |
+
first_frame_path = os.path.join(args.output_dir, f"first_frame_{str(uuid.uuid4())[:4]}.png")
|
585 |
+
|
586 |
+
image_pil.save(first_frame_path)
|
587 |
+
|
588 |
+
return first_frame_path, first_frame_path, gr.State([])
|
589 |
+
|
590 |
+
|
591 |
+
def preprocess_image_end(image_end):
|
592 |
+
|
593 |
+
image_end_pil = image2pil(image_end.name)
|
594 |
+
|
595 |
+
raw_w, raw_h = image_end_pil.size
|
596 |
+
# resize_ratio = max(512 / raw_w, 320 / raw_h)
|
597 |
+
# image_end_pil = image_end_pil.resize((int(raw_w * resize_ratio), int(raw_h * resize_ratio)), Image.BILINEAR)
|
598 |
+
# image_end_pil = transforms.CenterCrop((320, 512))(image_end_pil.convert('RGB'))
|
599 |
+
image_end_pil = image_end_pil.resize((512, 320), Image.BILINEAR)
|
600 |
+
|
601 |
+
last_frame_path = os.path.join(args.output_dir, f"last_frame_{str(uuid.uuid4())[:4]}.png")
|
602 |
+
|
603 |
+
image_end_pil.save(last_frame_path)
|
604 |
+
|
605 |
+
return last_frame_path, last_frame_path, gr.State([])
|
606 |
+
|
607 |
+
|
608 |
+
def add_drag(tracking_points):
|
609 |
+
tracking_points.constructor_args['value'].append([])
|
610 |
+
return tracking_points
|
611 |
+
|
612 |
+
|
613 |
+
def delete_last_drag(tracking_points, first_frame_path, last_frame_path):
|
614 |
+
tracking_points.constructor_args['value'].pop()
|
615 |
+
transparent_background = Image.open(first_frame_path).convert('RGBA')
|
616 |
+
transparent_background_end = Image.open(last_frame_path).convert('RGBA')
|
617 |
+
w, h = transparent_background.size
|
618 |
+
transparent_layer = np.zeros((h, w, 4))
|
619 |
+
|
620 |
+
for track in tracking_points.constructor_args['value']:
|
621 |
+
if len(track) > 1:
|
622 |
+
for i in range(len(track)-1):
|
623 |
+
start_point = track[i]
|
624 |
+
end_point = track[i+1]
|
625 |
+
vx = end_point[0] - start_point[0]
|
626 |
+
vy = end_point[1] - start_point[1]
|
627 |
+
arrow_length = np.sqrt(vx**2 + vy**2)
|
628 |
+
if i == len(track)-2:
|
629 |
+
cv2.arrowedLine(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2, tipLength=8 / arrow_length)
|
630 |
+
else:
|
631 |
+
cv2.line(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2,)
|
632 |
+
else:
|
633 |
+
cv2.circle(transparent_layer, tuple(track[0]), 5, (255, 0, 0, 255), -1)
|
634 |
+
|
635 |
+
transparent_layer = Image.fromarray(transparent_layer.astype(np.uint8))
|
636 |
+
trajectory_map = Image.alpha_composite(transparent_background, transparent_layer)
|
637 |
+
trajectory_map_end = Image.alpha_composite(transparent_background_end, transparent_layer)
|
638 |
+
|
639 |
+
return tracking_points, trajectory_map, trajectory_map_end
|
640 |
+
|
641 |
+
|
642 |
+
def delete_last_step(tracking_points, first_frame_path, last_frame_path):
|
643 |
+
tracking_points.constructor_args['value'][-1].pop()
|
644 |
+
transparent_background = Image.open(first_frame_path).convert('RGBA')
|
645 |
+
transparent_background_end = Image.open(last_frame_path).convert('RGBA')
|
646 |
+
w, h = transparent_background.size
|
647 |
+
transparent_layer = np.zeros((h, w, 4))
|
648 |
+
|
649 |
+
for track in tracking_points.constructor_args['value']:
|
650 |
+
if len(track) > 1:
|
651 |
+
for i in range(len(track)-1):
|
652 |
+
start_point = track[i]
|
653 |
+
end_point = track[i+1]
|
654 |
+
vx = end_point[0] - start_point[0]
|
655 |
+
vy = end_point[1] - start_point[1]
|
656 |
+
arrow_length = np.sqrt(vx**2 + vy**2)
|
657 |
+
if i == len(track)-2:
|
658 |
+
cv2.arrowedLine(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2, tipLength=8 / arrow_length)
|
659 |
+
else:
|
660 |
+
cv2.line(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2,)
|
661 |
+
else:
|
662 |
+
cv2.circle(transparent_layer, tuple(track[0]), 5, (255, 0, 0, 255), -1)
|
663 |
+
|
664 |
+
transparent_layer = Image.fromarray(transparent_layer.astype(np.uint8))
|
665 |
+
trajectory_map = Image.alpha_composite(transparent_background, transparent_layer)
|
666 |
+
trajectory_map_end = Image.alpha_composite(transparent_background_end, transparent_layer)
|
667 |
+
|
668 |
+
return tracking_points, trajectory_map, trajectory_map_end
|
669 |
+
|
670 |
+
|
671 |
+
def add_tracking_points(tracking_points, first_frame_path, last_frame_path, evt: gr.SelectData): # SelectData is a subclass of EventData
|
672 |
+
print(f"You selected {evt.value} at {evt.index} from {evt.target}")
|
673 |
+
tracking_points.constructor_args['value'][-1].append(evt.index)
|
674 |
+
|
675 |
+
transparent_background = Image.open(first_frame_path).convert('RGBA')
|
676 |
+
transparent_background_end = Image.open(last_frame_path).convert('RGBA')
|
677 |
+
|
678 |
+
w, h = transparent_background.size
|
679 |
+
transparent_layer = 0
|
680 |
+
for idx, track in enumerate(tracking_points.constructor_args['value']):
|
681 |
+
# mask = cv2.imread(
|
682 |
+
# os.path.join(args.output_dir, f"mask_{idx+1}.jpg")
|
683 |
+
# )
|
684 |
+
mask = np.zeros((320, 512, 3))
|
685 |
+
color = color_list[idx+1]
|
686 |
+
transparent_layer = mask[:, :, 0].reshape(h, w, 1) * color.reshape(1, 1, -1) + transparent_layer
|
687 |
+
|
688 |
+
if len(track) > 1:
|
689 |
+
for i in range(len(track)-1):
|
690 |
+
start_point = track[i]
|
691 |
+
end_point = track[i+1]
|
692 |
+
vx = end_point[0] - start_point[0]
|
693 |
+
vy = end_point[1] - start_point[1]
|
694 |
+
arrow_length = np.sqrt(vx**2 + vy**2)
|
695 |
+
if i == len(track)-2:
|
696 |
+
cv2.arrowedLine(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2, tipLength=8 / arrow_length)
|
697 |
+
else:
|
698 |
+
cv2.line(transparent_layer, tuple(start_point), tuple(end_point), (255, 0, 0, 255), 2,)
|
699 |
+
else:
|
700 |
+
cv2.circle(transparent_layer, tuple(track[0]), 5, (255, 0, 0, 255), -1)
|
701 |
+
|
702 |
+
transparent_layer = Image.fromarray(transparent_layer.astype(np.uint8))
|
703 |
+
alpha_coef = 0.99
|
704 |
+
im2_data = transparent_layer.getdata()
|
705 |
+
new_im2_data = [(r, g, b, int(a * alpha_coef)) for r, g, b, a in im2_data]
|
706 |
+
transparent_layer.putdata(new_im2_data)
|
707 |
+
|
708 |
+
trajectory_map = Image.alpha_composite(transparent_background, transparent_layer)
|
709 |
+
trajectory_map_end = Image.alpha_composite(transparent_background_end, transparent_layer)
|
710 |
+
|
711 |
+
return tracking_points, trajectory_map, trajectory_map_end
|
712 |
+
|
713 |
+
with gr.Row():
|
714 |
+
with gr.Column(scale=1):
|
715 |
+
image_upload_button = gr.UploadButton(label="Upload Start Image", file_types=["image"])
|
716 |
+
image_end_upload_button = gr.UploadButton(label="Upload End Image", file_types=["image"])
|
717 |
+
# select_area_button = gr.Button(value="Select Area with SAM")
|
718 |
+
add_drag_button = gr.Button(value="Add New Drag Trajectory")
|
719 |
+
reset_button = gr.Button(value="Reset")
|
720 |
+
run_button = gr.Button(value="Run")
|
721 |
+
delete_last_drag_button = gr.Button(value="Delete last drag")
|
722 |
+
delete_last_step_button = gr.Button(value="Delete last step")
|
723 |
+
|
724 |
+
with gr.Column(scale=7):
|
725 |
+
with gr.Row():
|
726 |
+
with gr.Column(scale=6):
|
727 |
+
input_image = gr.Image(
|
728 |
+
label="start frame",
|
729 |
+
interactive=True,
|
730 |
+
height=320,
|
731 |
+
width=512,
|
732 |
+
)
|
733 |
+
|
734 |
+
with gr.Column(scale=6):
|
735 |
+
input_image_end = gr.Image(
|
736 |
+
label="end frame",
|
737 |
+
interactive=True,
|
738 |
+
height=320,
|
739 |
+
width=512,
|
740 |
+
)
|
741 |
+
|
742 |
+
with gr.Row():
|
743 |
+
with gr.Column(scale=1):
|
744 |
+
|
745 |
+
controlnet_cond_scale = gr.Slider(
|
746 |
+
label='Control Scale',
|
747 |
+
minimum=0.0,
|
748 |
+
maximum=10,
|
749 |
+
step=0.1,
|
750 |
+
value=1.0,
|
751 |
+
)
|
752 |
+
|
753 |
+
motion_bucket_id = gr.Slider(
|
754 |
+
label='Motion Bucket',
|
755 |
+
minimum=1,
|
756 |
+
maximum=180,
|
757 |
+
step=1,
|
758 |
+
value=100,
|
759 |
+
)
|
760 |
+
|
761 |
+
with gr.Column(scale=5):
|
762 |
+
output_video = gr.Image(
|
763 |
+
label="Output Video",
|
764 |
+
height=320,
|
765 |
+
width=1152,
|
766 |
+
)
|
767 |
+
|
768 |
+
|
769 |
+
with gr.Row():
|
770 |
+
gr.Markdown("""
|
771 |
+
## Citation
|
772 |
+
```bibtex
|
773 |
+
@article{wang2024framer,
|
774 |
+
title={Framer: Interactive Frame Interpolation},
|
775 |
+
author={Wang, Wen and Wang, Qiuyu and Zheng, Kecheng and Ouyang, Hao and Chen, Zhekai and Gong, Biao and Chen, Hao and Shen, Yujun and Shen, Chunhua},
|
776 |
+
journal={arXiv preprint https://arxiv.org/abs/2410.18978},
|
777 |
+
year={2024}
|
778 |
+
}
|
779 |
+
```
|
780 |
+
""")
|
781 |
+
|
782 |
+
image_upload_button.upload(preprocess_image, image_upload_button, [input_image, first_frame_path, tracking_points])
|
783 |
+
|
784 |
+
image_end_upload_button.upload(preprocess_image_end, image_end_upload_button, [input_image_end, last_frame_path, tracking_points])
|
785 |
+
|
786 |
+
add_drag_button.click(add_drag, tracking_points, [tracking_points, ])
|
787 |
+
|
788 |
+
delete_last_drag_button.click(delete_last_drag, [tracking_points, first_frame_path, last_frame_path], [tracking_points, input_image, input_image_end])
|
789 |
+
|
790 |
+
delete_last_step_button.click(delete_last_step, [tracking_points, first_frame_path, last_frame_path], [tracking_points, input_image, input_image_end])
|
791 |
+
|
792 |
+
reset_button.click(reset_states, [first_frame_path, last_frame_path, tracking_points], [first_frame_path, last_frame_path, tracking_points])
|
793 |
+
|
794 |
+
input_image.select(add_tracking_points, [tracking_points, first_frame_path, last_frame_path], [tracking_points, input_image, input_image_end])
|
795 |
+
|
796 |
+
input_image_end.select(add_tracking_points, [tracking_points, first_frame_path, last_frame_path], [tracking_points, input_image, input_image_end])
|
797 |
+
|
798 |
+
run_button.click(Framer.run, [first_frame_path, last_frame_path, tracking_points, controlnet_cond_scale, motion_bucket_id], output_video)
|
799 |
+
|
800 |
+
demo.launch()
|