 |
+  |
+  |
+
| Original video | +"Jessica Chastain" | +"Watercolor painting" | +
 |
+  |
+  |
+
| "close up of a flower" | +"bonfire near the camp in the mountains at night" | +"close up of a diamond laying on the table" | +
 |
+  |
+  |
+
| "close up of macaroni on the plate" | +"close up of golden sphere" | +"a tree standing in the winter forest" | +
|
+ |
+ |
+
| Original video | +"Jessica Chastain" | +"Watercolor painting" | +
|
+ |
+ |
+
| "close up of a flower" | +"bonfire near the camp in the mountains at night" | +"close up of a diamond laying on the table" | +
|
+ |
+ |
+
| "close up of macaroni on the plate" | +"close up of golden sphere" | +"a tree standing in the winter forest" | +
') + v2v_step_1_blend_alpha = gr.Slider(label='Warped prev frame vs Current frame blend alpha', minimum=0, maximum=1, step=0.1, value=1, interactive=True) + v2v_step_1_processing_mode = gr.Radio(["Process full image then blend in occlusions", "Inpaint occlusions"], type="index", \ + label="Processing mode", value="Process full image then blend in occlusions", interactive=True) + + + with gr.Column(scale=1, variant='compact'): + gr.HTML('# Step 2 params:') + v2v_step_2_seed = gr.Number(label='Seed', value = 8888, Interactive = True, precision=0) + + with FormRow(elem_id="vid2vid_override_settings_row") as row: + v2v_override_settings = create_override_settings_dropdown("vid2vid", row) + + with FormGroup(elem_id=f"script_container"): + v2v_custom_inputs = scripts.scripts_img2img.setup_ui() + + with gr.Tab('txt2vid') as tab_txt2vid: + with gr.Row(): + gr.HTML('Control video (each frame will be used as input image to CN): *NOT REQUIRED') + with gr.Row(): + t2v_file = gr.File(label="Input video", interactive=True, file_count="single", file_types=["video"], elem_id="tex_to_vid_chosen_file") + t2v_init_image = gr.Image(label="Input image", interactive=True, file_count="single", file_types=["image"], elem_id="tex_to_vid_init_image") + + t2v_width, t2v_height, t2v_prompt, t2v_n_prompt, t2v_cfg_scale, t2v_seed, t2v_processing_strength, t2v_fix_frame_strength, t2v_sampler_index, t2v_steps = setup_common_values('txt2vid', t2v_args) + + with gr.Row(): + t2v_length = gr.Slider(label='Length (in frames)', minimum=10, maximum=2048, step=10, value=40, interactive=True) + t2v_fps = gr.Slider(label='Video FPS', minimum=4, maximum=64, step=4, value=12, interactive=True) + + gr.HTML('
') + t2v_cn_frame_send = gr.Radio(["None", "Current generated frame", "Previous generated frame", "Current reference video frame"], type="index", \ + label="What frame should be send to CN?", value="None", interactive=True) + + with FormRow(elem_id="txt2vid_override_settings_row") as row: + t2v_override_settings = create_override_settings_dropdown("txt2vid", row) + + with FormGroup(elem_id=f"script_container"): + t2v_custom_inputs = scripts.scripts_txt2img.setup_ui() + + tab_vid2vid.select(fn=lambda: 'vid2vid', inputs=[], outputs=[glo_sdcn_process_mode]) + tab_txt2vid.select(fn=lambda: 'txt2vid', inputs=[], outputs=[glo_sdcn_process_mode]) + + return locals() + +def process(*args): + msg = 'Done' + try: + if args[0] == 'vid2vid': + yield from vid2vid.start_process(*args) + elif args[0] == 'txt2vid': + yield from txt2vid.start_process(*args) + else: + msg = f"Unsupported processing mode: '{args[0]}'" + raise Exception(msg) + except Exception as error: + # handle the exception + msg = f"An exception occurred while trying to process the frame: {error}" + print(msg) + traceback.print_exc() + + yield msg, gr.Image.update(), gr.Image.update(), gr.Image.update(), gr.Image.update(), gr.Video.update(), gr.Button.update(interactive=True), gr.Button.update(interactive=False) + +def stop_process(*args): + utils.shared.is_interrupted = True + return gr.Button.update(interactive=False) + + + +def on_ui_tabs(): + modules.scripts.scripts_current = modules.scripts.scripts_img2img + modules.scripts.scripts_img2img.initialize_scripts(is_img2img=True) + + with gr.Blocks(analytics_enabled=False) as sdcnanim_interface: + components = {} + + #dv = SimpleNamespace(**T2VOutputArgs()) + with gr.Row(elem_id='sdcn-core').style(equal_height=False, variant='compact'): + with gr.Column(scale=1, variant='panel'): + #with gr.Tabs(): + components = inputs_ui() + + with gr.Accordion("Export settings", open=False): + export_settings_button = gr.Button('Export', elem_id=f"sdcn_export_settings_button") + export_setting_json = gr.Code(value='') + + + with gr.Column(scale=1, variant='compact'): + with gr.Row(variant='compact'): + run_button = gr.Button('Generate', elem_id=f"sdcn_anim_generate", variant='primary') + stop_button = gr.Button('Interrupt', elem_id=f"sdcn_anim_interrupt", variant='primary', interactive=False) + + save_frames_check = gr.Checkbox(label="Save frames into a folder nearby a video (check it before running the generation if you also want to save frames separately)", value=True, interactive=True) + gr.HTML('
') + + with gr.Column(variant="panel"): + sp_progress = gr.HTML(elem_id="sp_progress", value="") + + with gr.Row(variant='compact'): + img_preview_curr_frame = gr.Image(label='Current frame', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) + img_preview_curr_occl = gr.Image(label='Current occlusion', elem_id=f"img_preview_curr_occl", type='pil').style(height=240) + with gr.Row(variant='compact'): + img_preview_prev_warp = gr.Image(label='Previous frame warped', elem_id=f"img_preview_curr_frame", type='pil').style(height=240) + img_preview_processed = gr.Image(label='Processed', elem_id=f"img_preview_processed", type='pil').style(height=240) + + video_preview = gr.Video(interactive=False) + + with gr.Row(variant='compact'): + dummy_component = gr.Label(visible=False) + + components['glo_save_frames_check'] = save_frames_check + + # Define parameters for the action methods. + utils.shared.v2v_custom_inputs_size = len(components['v2v_custom_inputs']) + utils.shared.t2v_custom_inputs_size = len(components['t2v_custom_inputs']) + #print('v2v_custom_inputs', len(components['v2v_custom_inputs']), components['v2v_custom_inputs']) + #print('t2v_custom_inputs', len(components['t2v_custom_inputs']), components['t2v_custom_inputs']) + method_inputs = [components[name] for name in utils.get_component_names()] + components['v2v_custom_inputs'] + components['t2v_custom_inputs'] + + method_outputs = [ + sp_progress, + img_preview_curr_frame, + img_preview_curr_occl, + img_preview_prev_warp, + img_preview_processed, + video_preview, + run_button, + stop_button, + ] + + run_button.click( + fn=process, #wrap_gradio_gpu_call(start_process, extra_outputs=[None, '', '']), + inputs=method_inputs, + outputs=method_outputs, + show_progress=True, + ) + + stop_button.click( + fn=stop_process, + outputs=[stop_button], + show_progress=False + ) + + export_settings_button.click( + fn=utils.export_settings, + inputs=method_inputs, + outputs=[export_setting_json], + show_progress=False + ) + + modules.scripts.scripts_current = None + + # define queue - required for generators + sdcnanim_interface.queue(concurrency_count=1) + return [(sdcnanim_interface, "SD-CN-Animation", "sd_cn_animation_interface")] + + +script_callbacks.on_ui_tabs(on_ui_tabs) diff --git a/SD-CN-Animation/scripts/core/__pycache__/flow_utils.cpython-310.pyc b/SD-CN-Animation/scripts/core/__pycache__/flow_utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6366601ea0662901491dd85d2ffae5b130d265b5 Binary files /dev/null and b/SD-CN-Animation/scripts/core/__pycache__/flow_utils.cpython-310.pyc differ diff --git a/SD-CN-Animation/scripts/core/__pycache__/txt2vid.cpython-310.pyc b/SD-CN-Animation/scripts/core/__pycache__/txt2vid.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e170d95c657d927f42e76e5d15583ba5b701b9d Binary files /dev/null and b/SD-CN-Animation/scripts/core/__pycache__/txt2vid.cpython-310.pyc differ diff --git a/SD-CN-Animation/scripts/core/__pycache__/utils.cpython-310.pyc b/SD-CN-Animation/scripts/core/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f5590bd68baf60d36ef3060bc361aab2ee34600 Binary files /dev/null and b/SD-CN-Animation/scripts/core/__pycache__/utils.cpython-310.pyc differ diff --git a/SD-CN-Animation/scripts/core/__pycache__/vid2vid.cpython-310.pyc b/SD-CN-Animation/scripts/core/__pycache__/vid2vid.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb3030945fa8b85ea580730d1e6ef8136e23942a Binary files /dev/null and b/SD-CN-Animation/scripts/core/__pycache__/vid2vid.cpython-310.pyc differ diff --git a/SD-CN-Animation/scripts/core/flow_utils.py b/SD-CN-Animation/scripts/core/flow_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15eae7732c1200d392a15fcdc1015991fc63cb5e --- /dev/null +++ b/SD-CN-Animation/scripts/core/flow_utils.py @@ -0,0 +1,156 @@ +import sys, os + +import numpy as np +import cv2 + +from collections import namedtuple +import torch +import argparse +from RAFT.raft import RAFT +from RAFT.utils.utils import InputPadder + +import modules.paths as ph +import gc + +RAFT_model = None +fgbg = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + +def background_subtractor(frame, fgbg): + fgmask = fgbg.apply(frame) + return cv2.bitwise_and(frame, frame, mask=fgmask) + +def RAFT_clear_memory(): + global RAFT_model + del RAFT_model + gc.collect() + torch.cuda.empty_cache() + RAFT_model = None + +def RAFT_estimate_flow(frame1, frame2, device='cuda'): + global RAFT_model + + org_size = frame1.shape[1], frame1.shape[0] + size = frame1.shape[1] // 16 * 16, frame1.shape[0] // 16 * 16 + frame1 = cv2.resize(frame1, size) + frame2 = cv2.resize(frame2, size) + + model_path = ph.models_path + '/RAFT/raft-things.pth' + remote_model_path = 'https://drive.google.com/uc?id=1MqDajR89k-xLV0HIrmJ0k-n8ZpG6_suM' + + if not os.path.isfile(model_path): + from basicsr.utils.download_util import load_file_from_url + os.makedirs(os.path.dirname(model_path), exist_ok=True) + load_file_from_url(remote_model_path, file_name=model_path) + + if RAFT_model is None: + args = argparse.Namespace(**{ + 'model': ph.models_path + '/RAFT/raft-things.pth', + 'mixed_precision': True, + 'small': False, + 'alternate_corr': False, + 'path': "" + }) + + RAFT_model = torch.nn.DataParallel(RAFT(args)) + RAFT_model.load_state_dict(torch.load(args.model)) + + RAFT_model = RAFT_model.module + RAFT_model.to(device) + RAFT_model.eval() + + with torch.no_grad(): + frame1_torch = torch.from_numpy(frame1).permute(2, 0, 1).float()[None].to(device) + frame2_torch = torch.from_numpy(frame2).permute(2, 0, 1).float()[None].to(device) + + padder = InputPadder(frame1_torch.shape) + image1, image2 = padder.pad(frame1_torch, frame2_torch) + + # estimate optical flow + _, next_flow = RAFT_model(image1, image2, iters=20, test_mode=True) + _, prev_flow = RAFT_model(image2, image1, iters=20, test_mode=True) + + next_flow = next_flow[0].permute(1, 2, 0).cpu().numpy() + prev_flow = prev_flow[0].permute(1, 2, 0).cpu().numpy() + + fb_flow = next_flow + prev_flow + fb_norm = np.linalg.norm(fb_flow, axis=2) + + occlusion_mask = fb_norm[..., None].repeat(3, axis=-1) + + next_flow = cv2.resize(next_flow, org_size) + prev_flow = cv2.resize(prev_flow, org_size) + + return next_flow, prev_flow, occlusion_mask + +def compute_diff_map(next_flow, prev_flow, prev_frame, cur_frame, prev_frame_styled, args_dict): + h, w = cur_frame.shape[:2] + fl_w, fl_h = next_flow.shape[:2] + + # normalize flow + next_flow = next_flow / np.array([fl_h,fl_w]) + prev_flow = prev_flow / np.array([fl_h,fl_w]) + + # compute occlusion mask + fb_flow = next_flow + prev_flow + fb_norm = np.linalg.norm(fb_flow , axis=2) + + zero_flow_mask = np.clip(1 - np.linalg.norm(prev_flow, axis=-1)[...,None] * 20, 0, 1) + diff_mask_flow = fb_norm[..., None] * zero_flow_mask + + # resize flow + next_flow = cv2.resize(next_flow, (w, h)) + next_flow = (next_flow * np.array([h,w])).astype(np.float32) + prev_flow = cv2.resize(prev_flow, (w, h)) + prev_flow = (prev_flow * np.array([h,w])).astype(np.float32) + + # Generate sampling grids + grid_y, grid_x = torch.meshgrid(torch.arange(0, h), torch.arange(0, w)) + flow_grid = torch.stack((grid_x, grid_y), dim=0).float() + flow_grid += torch.from_numpy(prev_flow).permute(2, 0, 1) + flow_grid = flow_grid.unsqueeze(0) + flow_grid[:, 0, :, :] = 2 * flow_grid[:, 0, :, :] / (w - 1) - 1 + flow_grid[:, 1, :, :] = 2 * flow_grid[:, 1, :, :] / (h - 1) - 1 + flow_grid = flow_grid.permute(0, 2, 3, 1) + + + prev_frame_torch = torch.from_numpy(prev_frame).float().unsqueeze(0).permute(0, 3, 1, 2) #N, C, H, W + prev_frame_styled_torch = torch.from_numpy(prev_frame_styled).float().unsqueeze(0).permute(0, 3, 1, 2) #N, C, H, W + + warped_frame = torch.nn.functional.grid_sample(prev_frame_torch, flow_grid, mode="nearest", padding_mode="reflection", align_corners=True).permute(0, 2, 3, 1)[0].numpy() + warped_frame_styled = torch.nn.functional.grid_sample(prev_frame_styled_torch, flow_grid, mode="nearest", padding_mode="reflection", align_corners=True).permute(0, 2, 3, 1)[0].numpy() + + #warped_frame = cv2.remap(prev_frame, flow_map, None, cv2.INTER_NEAREST, borderMode = cv2.BORDER_REFLECT) + #warped_frame_styled = cv2.remap(prev_frame_styled, flow_map, None, cv2.INTER_NEAREST, borderMode = cv2.BORDER_REFLECT) + + + diff_mask_org = np.abs(warped_frame.astype(np.float32) - cur_frame.astype(np.float32)) / 255 + diff_mask_org = diff_mask_org.max(axis = -1, keepdims=True) + + diff_mask_stl = np.abs(warped_frame_styled.astype(np.float32) - cur_frame.astype(np.float32)) / 255 + diff_mask_stl = diff_mask_stl.max(axis = -1, keepdims=True) + + alpha_mask = np.maximum.reduce([diff_mask_flow * args_dict['occlusion_mask_flow_multiplier'] * 10, \ + diff_mask_org * args_dict['occlusion_mask_difo_multiplier'], \ + diff_mask_stl * args_dict['occlusion_mask_difs_multiplier']]) # + alpha_mask = alpha_mask.repeat(3, axis = -1) + + #alpha_mask_blured = cv2.dilate(alpha_mask, np.ones((5, 5), np.float32)) + if args_dict['occlusion_mask_blur'] > 0: + blur_filter_size = min(w,h) // 15 | 1 + alpha_mask = cv2.GaussianBlur(alpha_mask, (blur_filter_size, blur_filter_size) , args_dict['occlusion_mask_blur'], cv2.BORDER_REFLECT) + + alpha_mask = np.clip(alpha_mask, 0, 1) + + return alpha_mask, warped_frame_styled + +def frames_norm(frame): return frame / 127.5 - 1 + +def flow_norm(flow): return flow / 255 + +def occl_norm(occl): return occl / 127.5 - 1 + +def frames_renorm(frame): return (frame + 1) * 127.5 + +def flow_renorm(flow): return flow * 255 + +def occl_renorm(occl): return (occl + 1) * 127.5 diff --git a/SD-CN-Animation/scripts/core/txt2vid.py b/SD-CN-Animation/scripts/core/txt2vid.py new file mode 100644 index 0000000000000000000000000000000000000000..a03784a531e47d9e0176aa084ad1c21710888ef5 --- /dev/null +++ b/SD-CN-Animation/scripts/core/txt2vid.py @@ -0,0 +1,240 @@ +import sys, os + +import torch +import gc +import numpy as np +from PIL import Image + +import modules.paths as ph +from modules.shared import devices + +from scripts.core import utils, flow_utils +from FloweR.model import FloweR + +import skimage +import datetime +import cv2 +import gradio as gr +import time + +FloweR_model = None +DEVICE = 'cpu' +def FloweR_clear_memory(): + global FloweR_model + del FloweR_model + gc.collect() + torch.cuda.empty_cache() + FloweR_model = None + +def FloweR_load_model(w, h): + global DEVICE, FloweR_model + DEVICE = devices.get_optimal_device() + + model_path = ph.models_path + '/FloweR/FloweR_0.1.2.pth' + #remote_model_path = 'https://drive.google.com/uc?id=1K7gXUosgxU729_l-osl1HBU5xqyLsALv' #FloweR_0.1.1.pth + remote_model_path = 'https://drive.google.com/uc?id=1-UYsTXkdUkHLgtPK1Y5_7kKzCgzL_Z6o' #FloweR_0.1.2.pth + + if not os.path.isfile(model_path): + from basicsr.utils.download_util import load_file_from_url + os.makedirs(os.path.dirname(model_path), exist_ok=True) + load_file_from_url(remote_model_path, file_name=model_path) + + + FloweR_model = FloweR(input_size = (h, w)) + FloweR_model.load_state_dict(torch.load(model_path, map_location=DEVICE)) + # Move the model to the device + FloweR_model = FloweR_model.to(DEVICE) + FloweR_model.eval() + +def read_frame_from_video(input_video): + if input_video is None: return None + + # Reading video file + if input_video.isOpened(): + ret, cur_frame = input_video.read() + if cur_frame is not None: + cur_frame = cv2.cvtColor(cur_frame, cv2.COLOR_BGR2RGB) + else: + cur_frame = None + input_video.release() + input_video = None + + return cur_frame + +def start_process(*args): + processing_start_time = time.time() + args_dict = utils.args_to_dict(*args) + args_dict = utils.get_mode_args('t2v', args_dict) + + # Open the input video file + input_video = None + if args_dict['file'] is not None: + input_video = cv2.VideoCapture(args_dict['file'].name) + + # Create an output video file with the same fps, width, and height as the input video + output_video_name = f'outputs/sd-cn-animation/txt2vid/{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.mp4' + output_video_folder = os.path.splitext(output_video_name)[0] + os.makedirs(os.path.dirname(output_video_name), exist_ok=True) + + #if args_dict['save_frames_check']: + os.makedirs(output_video_folder, exist_ok=True) + + # Writing to current params to params.json + setts_json = utils.export_settings(*args) + with open(os.path.join(output_video_folder, "params.json"), "w") as outfile: + outfile.write(setts_json) + + curr_frame = None + prev_frame = None + + def save_result_to_image(image, ind): + if args_dict['save_frames_check']: + cv2.imwrite(os.path.join(output_video_folder, f'{ind:05d}.png'), cv2.cvtColor(image, cv2.COLOR_RGB2BGR)) + + def set_cn_frame_input(): + if args_dict['cn_frame_send'] == 0: # Current generated frame" + pass + elif args_dict['cn_frame_send'] == 1: # Current generated frame" + if curr_frame is not None: + utils.set_CNs_input_image(args_dict, Image.fromarray(curr_frame), set_references=True) + elif args_dict['cn_frame_send'] == 2: # Previous generated frame + if prev_frame is not None: + utils.set_CNs_input_image(args_dict, Image.fromarray(prev_frame), set_references=True) + elif args_dict['cn_frame_send'] == 3: # Current reference video frame + if input_video is not None: + curr_video_frame = read_frame_from_video(input_video) + curr_video_frame = cv2.resize(curr_video_frame, (args_dict['width'], args_dict['height'])) + utils.set_CNs_input_image(args_dict, Image.fromarray(curr_video_frame), set_references=True) + else: + raise Exception('There is no input video! Set it up first.') + else: + raise Exception('Incorrect cn_frame_send mode!') + + set_cn_frame_input() + + if args_dict['init_image'] is not None: + #resize array to args_dict['width'], args_dict['height'] + image_array=args_dict['init_image']#this is a numpy array + init_frame = np.array(Image.fromarray(image_array).resize((args_dict['width'], args_dict['height'])).convert('RGB')) + processed_frame = init_frame.copy() + else: + processed_frames, _, _, _ = utils.txt2img(args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + #if input_video is not None: + # processed_frame = skimage.exposure.match_histograms(processed_frame, curr_video_frame, channel_axis=-1) + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + init_frame = processed_frame.copy() + + output_video = cv2.VideoWriter(output_video_name, cv2.VideoWriter_fourcc(*'mp4v'), args_dict['fps'], (args_dict['width'], args_dict['height'])) + output_video.write(cv2.cvtColor(processed_frame, cv2.COLOR_RGB2BGR)) + + stat = f"Frame: 1 / {args_dict['length']}; " + utils.get_time_left(1, args_dict['length'], processing_start_time) + utils.shared.is_interrupted = False + + save_result_to_image(processed_frame, 1) + yield stat, init_frame, None, None, processed_frame, None, gr.Button.update(interactive=False), gr.Button.update(interactive=True) + + org_size = args_dict['width'], args_dict['height'] + size = args_dict['width'] // 128 * 128, args_dict['height'] // 128 * 128 + FloweR_load_model(size[0], size[1]) + + clip_frames = np.zeros((4, size[1], size[0], 3), dtype=np.uint8) + + prev_frame = init_frame + + for ind in range(args_dict['length'] - 1): + if utils.shared.is_interrupted: break + + args_dict = utils.args_to_dict(*args) + args_dict = utils.get_mode_args('t2v', args_dict) + + clip_frames = np.roll(clip_frames, -1, axis=0) + clip_frames[-1] = cv2.resize(prev_frame[...,:3], size) + clip_frames_torch = flow_utils.frames_norm(torch.from_numpy(clip_frames).to(DEVICE, dtype=torch.float32)) + + with torch.no_grad(): + pred_data = FloweR_model(clip_frames_torch.unsqueeze(0))[0] + + pred_flow = flow_utils.flow_renorm(pred_data[...,:2]).cpu().numpy() + pred_occl = flow_utils.occl_renorm(pred_data[...,2:3]).cpu().numpy().repeat(3, axis = -1) + pred_next = flow_utils.frames_renorm(pred_data[...,3:6]).cpu().numpy() + + pred_occl = np.clip(pred_occl * 10, 0, 255).astype(np.uint8) + pred_next = np.clip(pred_next, 0, 255).astype(np.uint8) + + pred_flow = cv2.resize(pred_flow, org_size) + pred_occl = cv2.resize(pred_occl, org_size) + pred_next = cv2.resize(pred_next, org_size) + + curr_frame = pred_next.copy() + + ''' + pred_flow = pred_flow / (1 + np.linalg.norm(pred_flow, axis=-1, keepdims=True) * 0.05) + pred_flow = cv2.GaussianBlur(pred_flow, (31,31), 1, cv2.BORDER_REFLECT_101) + + pred_occl = cv2.GaussianBlur(pred_occl, (21,21), 2, cv2.BORDER_REFLECT_101) + pred_occl = (np.abs(pred_occl / 255) ** 1.5) * 255 + pred_occl = np.clip(pred_occl * 25, 0, 255).astype(np.uint8) + + flow_map = pred_flow.copy() + flow_map[:,:,0] += np.arange(args_dict['width']) + flow_map[:,:,1] += np.arange(args_dict['height'])[:,np.newaxis] + + warped_frame = cv2.remap(prev_frame, flow_map, None, cv2.INTER_NEAREST, borderMode = cv2.BORDER_REFLECT_101) + alpha_mask = pred_occl / 255. + #alpha_mask = np.clip(alpha_mask + np.random.normal(0, 0.4, size = alpha_mask.shape), 0, 1) + curr_frame = pred_next.astype(float) * alpha_mask + warped_frame.astype(float) * (1 - alpha_mask) + curr_frame = np.clip(curr_frame, 0, 255).astype(np.uint8) + #curr_frame = warped_frame.copy() + ''' + + set_cn_frame_input() + + args_dict['mode'] = 4 + args_dict['init_img'] = Image.fromarray(pred_next) + args_dict['mask_img'] = Image.fromarray(pred_occl) + args_dict['seed'] = -1 + args_dict['denoising_strength'] = args_dict['processing_strength'] + + processed_frames, _, _, _ = utils.img2img(args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + #if input_video is not None: + # processed_frame = skimage.exposure.match_histograms(processed_frame, curr_video_frame, channel_axis=-1) + #else: + processed_frame = skimage.exposure.match_histograms(processed_frame, init_frame, channel_axis=-1) + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + + args_dict['mode'] = 0 + args_dict['init_img'] = Image.fromarray(processed_frame) + args_dict['mask_img'] = None + args_dict['seed'] = -1 + args_dict['denoising_strength'] = args_dict['fix_frame_strength'] + + #utils.set_CNs_input_image(args_dict, Image.fromarray(curr_frame)) + processed_frames, _, _, _ = utils.img2img(args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + #if input_video is not None: + # processed_frame = skimage.exposure.match_histograms(processed_frame, curr_video_frame, channel_axis=-1) + #else: + processed_frame = skimage.exposure.match_histograms(processed_frame, init_frame, channel_axis=-1) + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + + output_video.write(cv2.cvtColor(processed_frame, cv2.COLOR_RGB2BGR)) + prev_frame = processed_frame.copy() + + save_result_to_image(processed_frame, ind + 2) + stat = f"Frame: {ind + 2} / {args_dict['length']}; " + utils.get_time_left(ind+2, args_dict['length'], processing_start_time) + yield stat, curr_frame, pred_occl, pred_next, processed_frame, None, gr.Button.update(interactive=False), gr.Button.update(interactive=True) + + if input_video is not None: input_video.release() + output_video.release() + FloweR_clear_memory() + + curr_frame = gr.Image.update() + occlusion_mask = gr.Image.update() + warped_styled_frame_ = gr.Image.update() + processed_frame = gr.Image.update() + + # print('TOTAL TIME:', int(time.time() - processing_start_time)) + + yield 'done', curr_frame, occlusion_mask, warped_styled_frame_, processed_frame, output_video_name, gr.Button.update(interactive=True), gr.Button.update(interactive=False) \ No newline at end of file diff --git a/SD-CN-Animation/scripts/core/utils.py b/SD-CN-Animation/scripts/core/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ef3bcb90cc98550fb1c232f664972146d71846f9 --- /dev/null +++ b/SD-CN-Animation/scripts/core/utils.py @@ -0,0 +1,432 @@ +class shared: + is_interrupted = False + v2v_custom_inputs_size = 0 + t2v_custom_inputs_size = 0 + +def get_component_names(): + components_list = [ + 'glo_sdcn_process_mode', + 'v2v_file', 'v2v_width', 'v2v_height', 'v2v_prompt', 'v2v_n_prompt', 'v2v_cfg_scale', 'v2v_seed', 'v2v_processing_strength', 'v2v_fix_frame_strength', + 'v2v_sampler_index', 'v2v_steps', 'v2v_override_settings', + 'v2v_occlusion_mask_blur', 'v2v_occlusion_mask_trailing', 'v2v_occlusion_mask_flow_multiplier', 'v2v_occlusion_mask_difo_multiplier', 'v2v_occlusion_mask_difs_multiplier', + 'v2v_step_1_processing_mode', 'v2v_step_1_blend_alpha', 'v2v_step_1_seed', 'v2v_step_2_seed', + 't2v_file','t2v_init_image', 't2v_width', 't2v_height', 't2v_prompt', 't2v_n_prompt', 't2v_cfg_scale', 't2v_seed', 't2v_processing_strength', 't2v_fix_frame_strength', + 't2v_sampler_index', 't2v_steps', 't2v_length', 't2v_fps', 't2v_cn_frame_send', + 'glo_save_frames_check' + ] + + return components_list + +def args_to_dict(*args): # converts list of argumets into dictionary for better handling of it + args_list = get_component_names() + + # set default values for params that were not specified + args_dict = { + # video to video params + 'v2v_mode': 0, + 'v2v_prompt': '', + 'v2v_n_prompt': '', + 'v2v_prompt_styles': [], + 'v2v_init_video': None, # Always required + + 'v2v_steps': 15, + 'v2v_sampler_index': 0, # 'Euler a' + 'v2v_mask_blur': 0, + + 'v2v_inpainting_fill': 1, # original + 'v2v_restore_faces': False, + 'v2v_tiling': False, + 'v2v_n_iter': 1, + 'v2v_batch_size': 1, + 'v2v_cfg_scale': 5.5, + 'v2v_image_cfg_scale': 1.5, + 'v2v_denoising_strength': 0.75, + 'v2v_processing_strength': 0.85, + 'v2v_fix_frame_strength': 0.15, + 'v2v_seed': -1, + 'v2v_subseed': -1, + 'v2v_subseed_strength': 0, + 'v2v_seed_resize_from_h': 512, + 'v2v_seed_resize_from_w': 512, + 'v2v_seed_enable_extras': False, + 'v2v_height': 512, + 'v2v_width': 512, + 'v2v_resize_mode': 1, + 'v2v_inpaint_full_res': True, + 'v2v_inpaint_full_res_padding': 0, + 'v2v_inpainting_mask_invert': False, + + # text to video params + 't2v_mode': 4, + 't2v_prompt': '', + 't2v_n_prompt': '', + 't2v_prompt_styles': [], + 't2v_init_img': None, + 't2v_mask_img': None, + + 't2v_steps': 15, + 't2v_sampler_index': 0, # 'Euler a' + 't2v_mask_blur': 0, + + 't2v_inpainting_fill': 1, # original + 't2v_restore_faces': False, + 't2v_tiling': False, + 't2v_n_iter': 1, + 't2v_batch_size': 1, + 't2v_cfg_scale': 5.5, + 't2v_image_cfg_scale': 1.5, + 't2v_denoising_strength': 0.75, + 't2v_processing_strength': 0.85, + 't2v_fix_frame_strength': 0.15, + 't2v_seed': -1, + 't2v_subseed': -1, + 't2v_subseed_strength': 0, + 't2v_seed_resize_from_h': 512, + 't2v_seed_resize_from_w': 512, + 't2v_seed_enable_extras': False, + 't2v_height': 512, + 't2v_width': 512, + 't2v_resize_mode': 1, + 't2v_inpaint_full_res': True, + 't2v_inpaint_full_res_padding': 0, + 't2v_inpainting_mask_invert': False, + + 't2v_override_settings': [], + #'t2v_script_inputs': [0], + + 't2v_fps': 12, + } + + args = list(args) + + for i in range(len(args_list)): + if (args[i] is None) and (args_list[i] in args_dict): + #args[i] = args_dict[args_list[i]] + pass + else: + args_dict[args_list[i]] = args[i] + + args_dict['v2v_script_inputs'] = args[len(args_list):len(args_list)+shared.v2v_custom_inputs_size] + #print('v2v_script_inputs', args_dict['v2v_script_inputs']) + args_dict['t2v_script_inputs'] = args[len(args_list)+shared.v2v_custom_inputs_size:] + #print('t2v_script_inputs', args_dict['t2v_script_inputs']) + return args_dict + +def get_mode_args(mode, args_dict): + mode_args_dict = {} + for key, value in args_dict.items(): + if key[:3] in [mode, 'glo'] : + mode_args_dict[key[4:]] = value + + return mode_args_dict + +def set_CNs_input_image(args_dict, image, set_references = False): + for script_input in args_dict['script_inputs']: + if type(script_input).__name__ == 'UiControlNetUnit': + if script_input.module not in ["reference_only", "reference_adain", "reference_adain+attn"] or set_references: + script_input.image = np.array(image) + script_input.batch_images = [np.array(image)] + +import time +import datetime + +def get_time_left(ind, length, processing_start_time): + s_passed = int(time.time() - processing_start_time) + time_passed = datetime.timedelta(seconds=s_passed) + s_left = int(s_passed / ind * (length - ind)) + time_left = datetime.timedelta(seconds=s_left) + return f"Time elapsed: {time_passed}; Time left: {time_left};" + +import numpy as np +from PIL import Image, ImageOps, ImageFilter, ImageEnhance, ImageChops +from types import SimpleNamespace + +from modules.generation_parameters_copypaste import create_override_settings_dict +from modules.processing import Processed, StableDiffusionProcessingImg2Img, StableDiffusionProcessingTxt2Img, process_images +import modules.processing as processing +from modules.ui import plaintext_to_html +import modules.images as images +import modules.scripts +from modules.shared import opts, devices, state +from modules import devices, sd_samplers, img2img +from modules import shared, sd_hijack, lowvram + +# TODO: Refactor all the code below + +def process_img(p, input_img, output_dir, inpaint_mask_dir, args): + processing.fix_seed(p) + + #images = shared.listfiles(input_dir) + images = [input_img] + + is_inpaint_batch = False + #if inpaint_mask_dir: + # inpaint_masks = shared.listfiles(inpaint_mask_dir) + # is_inpaint_batch = len(inpaint_masks) > 0 + #if is_inpaint_batch: + # print(f"\nInpaint batch is enabled. {len(inpaint_masks)} masks found.") + + #print(f"Will process {len(images)} images, creating {p.n_iter * p.batch_size} new images for each.") + + save_normally = output_dir == '' + + p.do_not_save_grid = True + p.do_not_save_samples = not save_normally + + state.job_count = len(images) * p.n_iter + + generated_images = [] + for i, image in enumerate(images): + state.job = f"{i+1} out of {len(images)}" + if state.skipped: + state.skipped = False + + if state.interrupted: + break + + img = image #Image.open(image) + # Use the EXIF orientation of photos taken by smartphones. + img = ImageOps.exif_transpose(img) + p.init_images = [img] * p.batch_size + + #if is_inpaint_batch: + # # try to find corresponding mask for an image using simple filename matching + # mask_image_path = os.path.join(inpaint_mask_dir, os.path.basename(image)) + # # if not found use first one ("same mask for all images" use-case) + # if not mask_image_path in inpaint_masks: + # mask_image_path = inpaint_masks[0] + # mask_image = Image.open(mask_image_path) + # p.image_mask = mask_image + + proc = modules.scripts.scripts_img2img.run(p, *args) + if proc is None: + proc = process_images(p) + generated_images.append(proc.images[0]) + + #for n, processed_image in enumerate(proc.images): + # filename = os.path.basename(image) + + # if n > 0: + # left, right = os.path.splitext(filename) + # filename = f"{left}-{n}{right}" + + # if not save_normally: + # os.makedirs(output_dir, exist_ok=True) + # if processed_image.mode == 'RGBA': + # processed_image = processed_image.convert("RGB") + # processed_image.save(os.path.join(output_dir, filename)) + + return generated_images + +def img2img(args_dict): + args = SimpleNamespace(**args_dict) + override_settings = create_override_settings_dict(args.override_settings) + + is_batch = args.mode == 5 + + if args.mode == 0: # img2img + image = args.init_img.convert("RGB") + mask = None + elif args.mode == 1: # img2img sketch + image = args.sketch.convert("RGB") + mask = None + elif args.mode == 2: # inpaint + image, mask = args.init_img_with_mask["image"], args.init_img_with_mask["mask"] + alpha_mask = ImageOps.invert(image.split()[-1]).convert('L').point(lambda x: 255 if x > 0 else 0, mode='1') + mask = ImageChops.lighter(alpha_mask, mask.convert('L')).convert('L') + image = image.convert("RGB") + elif args.mode == 3: # inpaint sketch + image = args.inpaint_color_sketch + orig = args.inpaint_color_sketch_orig or args.inpaint_color_sketch + pred = np.any(np.array(image) != np.array(orig), axis=-1) + mask = Image.fromarray(pred.astype(np.uint8) * 255, "L") + mask = ImageEnhance.Brightness(mask).enhance(1 - args.mask_alpha / 100) + blur = ImageFilter.GaussianBlur(args.mask_blur) + image = Image.composite(image.filter(blur), orig, mask.filter(blur)) + image = image.convert("RGB") + elif args.mode == 4: # inpaint upload mask + #image = args.init_img_inpaint + #mask = args.init_mask_inpaint + + image = args.init_img.convert("RGB") + mask = args.mask_img.convert("L") + else: + image = None + mask = None + + # Use the EXIF orientation of photos taken by smartphones. + if image is not None: + image = ImageOps.exif_transpose(image) + + assert 0. <= args.denoising_strength <= 1., 'can only work with strength in [0.0, 1.0]' + + p = StableDiffusionProcessingImg2Img( + sd_model=shared.sd_model, + outpath_samples=opts.outdir_samples or opts.outdir_img2img_samples, + outpath_grids=opts.outdir_grids or opts.outdir_img2img_grids, + prompt=args.prompt, + negative_prompt=args.n_prompt, + styles=args.prompt_styles, + seed=args.seed, + subseed=args.subseed, + subseed_strength=args.subseed_strength, + seed_resize_from_h=args.seed_resize_from_h, + seed_resize_from_w=args.seed_resize_from_w, + seed_enable_extras=args.seed_enable_extras, + sampler_name=sd_samplers.samplers_for_img2img[args.sampler_index].name, + batch_size=args.batch_size, + n_iter=args.n_iter, + steps=args.steps, + cfg_scale=args.cfg_scale, + width=args.width, + height=args.height, + restore_faces=args.restore_faces, + tiling=args.tiling, + init_images=[image], + mask=mask, + mask_blur=args.mask_blur, + inpainting_fill=args.inpainting_fill, + resize_mode=args.resize_mode, + denoising_strength=args.denoising_strength, + image_cfg_scale=args.image_cfg_scale, + inpaint_full_res=args.inpaint_full_res, + inpaint_full_res_padding=args.inpaint_full_res_padding, + inpainting_mask_invert=args.inpainting_mask_invert, + override_settings=override_settings, + ) + + p.scripts = modules.scripts.scripts_img2img + p.script_args = args.script_inputs + + #if shared.cmd_opts.enable_console_prompts: + # print(f"\nimg2img: {args.prompt}", file=shared.progress_print_out) + + if mask: + p.extra_generation_params["Mask blur"] = args.mask_blur + + ''' + if is_batch: + ... + # assert not shared.cmd_opts.hide_ui_dir_config, "Launched with --hide-ui-dir-config, batch img2img disabled" + # process_batch(p, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, args.script_inputs) + # processed = Processed(p, [], p.seed, "") + else: + processed = modules.scripts.scripts_img2img.run(p, *args.script_inputs) + if processed is None: + processed = process_images(p) + ''' + + generated_images = process_img(p, image, None, '', args.script_inputs) + processed = Processed(p, [], p.seed, "") + p.close() + + shared.total_tqdm.clear() + + generation_info_js = processed.js() + #if opts.samples_log_stdout: + # print(generation_info_js) + + #if opts.do_not_show_images: + # processed.images = [] + + #print(generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments)) + return generated_images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) + +def txt2img(args_dict): + args = SimpleNamespace(**args_dict) + override_settings = create_override_settings_dict(args.override_settings) + + p = StableDiffusionProcessingTxt2Img( + sd_model=shared.sd_model, + outpath_samples=opts.outdir_samples or opts.outdir_txt2img_samples, + outpath_grids=opts.outdir_grids or opts.outdir_txt2img_grids, + prompt=args.prompt, + styles=args.prompt_styles, + negative_prompt=args.n_prompt, + seed=args.seed, + subseed=args.subseed, + subseed_strength=args.subseed_strength, + seed_resize_from_h=args.seed_resize_from_h, + seed_resize_from_w=args.seed_resize_from_w, + seed_enable_extras=args.seed_enable_extras, + sampler_name=sd_samplers.samplers[args.sampler_index].name, + batch_size=args.batch_size, + n_iter=args.n_iter, + steps=args.steps, + cfg_scale=args.cfg_scale, + width=args.width, + height=args.height, + restore_faces=args.restore_faces, + tiling=args.tiling, + #enable_hr=args.enable_hr, + #denoising_strength=args.denoising_strength if enable_hr else None, + #hr_scale=hr_scale, + #hr_upscaler=hr_upscaler, + #hr_second_pass_steps=hr_second_pass_steps, + #hr_resize_x=hr_resize_x, + #hr_resize_y=hr_resize_y, + override_settings=override_settings, + ) + + p.scripts = modules.scripts.scripts_txt2img + p.script_args = args.script_inputs + + #if cmd_opts.enable_console_prompts: + # print(f"\ntxt2img: {prompt}", file=shared.progress_print_out) + + processed = modules.scripts.scripts_txt2img.run(p, *args.script_inputs) + + if processed is None: + processed = process_images(p) + + p.close() + + shared.total_tqdm.clear() + + generation_info_js = processed.js() + #if opts.samples_log_stdout: + # print(generation_info_js) + + #if opts.do_not_show_images: + # processed.images = [] + + return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html(processed.comments) + + +import json +def get_json(obj): + return json.loads( + json.dumps(obj, default=lambda o: getattr(o, '__dict__', str(o))) + ) + +def export_settings(*args): + args_dict = args_to_dict(*args) + if args[0] == 'vid2vid': + args_dict = get_mode_args('v2v', args_dict) + elif args[0] == 'txt2vid': + args_dict = get_mode_args('t2v', args_dict) + else: + msg = f"Unsupported processing mode: '{args[0]}'" + raise Exception(msg) + + # convert CN params into a readable dict + cn_remove_list = ['low_vram', 'is_ui', 'input_mode', 'batch_images', 'output_dir', 'loopback', 'image'] + + args_dict['ControlNets'] = [] + for script_input in args_dict['script_inputs']: + if type(script_input).__name__ == 'UiControlNetUnit': + cn_values_dict = get_json(script_input) + if cn_values_dict['enabled']: + for key in cn_remove_list: + if key in cn_values_dict: del cn_values_dict[key] + args_dict['ControlNets'].append(cn_values_dict) + + # remove unimportant values + remove_list = ['save_frames_check', 'restore_faces', 'prompt_styles', 'mask_blur', 'inpainting_fill', 'tiling', 'n_iter', 'batch_size', 'subseed', 'subseed_strength', 'seed_resize_from_h', \ + 'seed_resize_from_w', 'seed_enable_extras', 'resize_mode', 'inpaint_full_res', 'inpaint_full_res_padding', 'inpainting_mask_invert', 'file', 'denoising_strength', \ + 'override_settings', 'script_inputs', 'init_img', 'mask_img', 'mode', 'init_video'] + + for key in remove_list: + if key in args_dict: del args_dict[key] + + return json.dumps(args_dict, indent=2, default=lambda o: getattr(o, '__dict__', str(o))) \ No newline at end of file diff --git a/SD-CN-Animation/scripts/core/vid2vid.py b/SD-CN-Animation/scripts/core/vid2vid.py new file mode 100644 index 0000000000000000000000000000000000000000..207af3060c55fe077bf1814a31c1c715fb3e23eb --- /dev/null +++ b/SD-CN-Animation/scripts/core/vid2vid.py @@ -0,0 +1,270 @@ +import sys, os + +import math +import os +import sys +import traceback + +import numpy as np +from PIL import Image + +from modules import devices, sd_samplers +from modules import shared, sd_hijack, lowvram + +from modules.shared import devices +import modules.shared as shared + +import gc +import cv2 +import gradio as gr + +import time +import skimage +import datetime + +from scripts.core.flow_utils import RAFT_estimate_flow, RAFT_clear_memory, compute_diff_map +from scripts.core import utils + +class sdcn_anim_tmp: + prepear_counter = 0 + process_counter = 0 + input_video = None + output_video = None + curr_frame = None + prev_frame = None + prev_frame_styled = None + prev_frame_alpha_mask = None + fps = None + total_frames = None + prepared_frames = None + prepared_next_flows = None + prepared_prev_flows = None + frames_prepared = False + +def read_frame_from_video(): + # Reading video file + if sdcn_anim_tmp.input_video.isOpened(): + ret, cur_frame = sdcn_anim_tmp.input_video.read() + if cur_frame is not None: + cur_frame = cv2.cvtColor(cur_frame, cv2.COLOR_BGR2RGB) + else: + cur_frame = None + sdcn_anim_tmp.input_video.release() + + return cur_frame + +def get_cur_stat(): + stat = f'Frames prepared: {sdcn_anim_tmp.prepear_counter + 1} / {sdcn_anim_tmp.total_frames}; ' + stat += f'Frames processed: {sdcn_anim_tmp.process_counter + 1} / {sdcn_anim_tmp.total_frames}; ' + return stat + +def clear_memory_from_sd(): + if shared.sd_model is not None: + sd_hijack.model_hijack.undo_hijack(shared.sd_model) + try: + lowvram.send_everything_to_cpu() + except Exception as e: + ... + del shared.sd_model + shared.sd_model = None + gc.collect() + devices.torch_gc() + +def start_process(*args): + processing_start_time = time.time() + args_dict = utils.args_to_dict(*args) + args_dict = utils.get_mode_args('v2v', args_dict) + + sdcn_anim_tmp.process_counter = 0 + sdcn_anim_tmp.prepear_counter = 0 + + # Open the input video file + sdcn_anim_tmp.input_video = cv2.VideoCapture(args_dict['file'].name) + + # Get useful info from the source video + sdcn_anim_tmp.fps = int(sdcn_anim_tmp.input_video.get(cv2.CAP_PROP_FPS)) + sdcn_anim_tmp.total_frames = int(sdcn_anim_tmp.input_video.get(cv2.CAP_PROP_FRAME_COUNT)) + loop_iterations = (sdcn_anim_tmp.total_frames-1) * 2 + + # Create an output video file with the same fps, width, and height as the input video + output_video_name = f'outputs/sd-cn-animation/vid2vid/{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.mp4' + output_video_folder = os.path.splitext(output_video_name)[0] + os.makedirs(os.path.dirname(output_video_name), exist_ok=True) + + if args_dict['save_frames_check']: + os.makedirs(output_video_folder, exist_ok=True) + + def save_result_to_image(image, ind): + if args_dict['save_frames_check']: + cv2.imwrite(os.path.join(output_video_folder, f'{ind:05d}.png'), cv2.cvtColor(image, cv2.COLOR_RGB2BGR)) + + sdcn_anim_tmp.output_video = cv2.VideoWriter(output_video_name, cv2.VideoWriter_fourcc(*'mp4v'), sdcn_anim_tmp.fps, (args_dict['width'], args_dict['height'])) + + curr_frame = read_frame_from_video() + curr_frame = cv2.resize(curr_frame, (args_dict['width'], args_dict['height'])) + sdcn_anim_tmp.prepared_frames = np.zeros((11, args_dict['height'], args_dict['width'], 3), dtype=np.uint8) + sdcn_anim_tmp.prepared_next_flows = np.zeros((10, args_dict['height'], args_dict['width'], 2)) + sdcn_anim_tmp.prepared_prev_flows = np.zeros((10, args_dict['height'], args_dict['width'], 2)) + sdcn_anim_tmp.prepared_frames[0] = curr_frame + + args_dict['init_img'] = Image.fromarray(curr_frame) + utils.set_CNs_input_image(args_dict, Image.fromarray(curr_frame)) + processed_frames, _, _, _ = utils.img2img(args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + processed_frame = skimage.exposure.match_histograms(processed_frame, curr_frame, channel_axis=None) + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + #print('Processed frame ', 0) + + sdcn_anim_tmp.curr_frame = curr_frame + sdcn_anim_tmp.prev_frame = curr_frame.copy() + sdcn_anim_tmp.prev_frame_styled = processed_frame.copy() + utils.shared.is_interrupted = False + + save_result_to_image(processed_frame, 1) + stat = get_cur_stat() + utils.get_time_left(1, loop_iterations, processing_start_time) + yield stat, sdcn_anim_tmp.curr_frame, None, None, processed_frame, None, gr.Button.update(interactive=False), gr.Button.update(interactive=True) + + for step in range(loop_iterations): + if utils.shared.is_interrupted: break + + args_dict = utils.args_to_dict(*args) + args_dict = utils.get_mode_args('v2v', args_dict) + + occlusion_mask = None + prev_frame = None + curr_frame = sdcn_anim_tmp.curr_frame + warped_styled_frame_ = gr.Image.update() + processed_frame = gr.Image.update() + + prepare_steps = 10 + if sdcn_anim_tmp.process_counter % prepare_steps == 0 and not sdcn_anim_tmp.frames_prepared: # prepare next 10 frames for processing + #clear_memory_from_sd() + device = devices.get_optimal_device() + + curr_frame = read_frame_from_video() + if curr_frame is not None: + curr_frame = cv2.resize(curr_frame, (args_dict['width'], args_dict['height'])) + prev_frame = sdcn_anim_tmp.prev_frame.copy() + + next_flow, prev_flow, occlusion_mask = RAFT_estimate_flow(prev_frame, curr_frame, device=device) + occlusion_mask = np.clip(occlusion_mask * 0.1 * 255, 0, 255).astype(np.uint8) + + cn = sdcn_anim_tmp.prepear_counter % 10 + if sdcn_anim_tmp.prepear_counter % 10 == 0: + sdcn_anim_tmp.prepared_frames[cn] = sdcn_anim_tmp.prev_frame + sdcn_anim_tmp.prepared_frames[cn + 1] = curr_frame.copy() + sdcn_anim_tmp.prepared_next_flows[cn] = next_flow.copy() + sdcn_anim_tmp.prepared_prev_flows[cn] = prev_flow.copy() + #print('Prepared frame ', cn+1) + + sdcn_anim_tmp.prev_frame = curr_frame.copy() + + sdcn_anim_tmp.prepear_counter += 1 + if sdcn_anim_tmp.prepear_counter % prepare_steps == 0 or \ + sdcn_anim_tmp.prepear_counter >= sdcn_anim_tmp.total_frames - 1 or \ + curr_frame is None: + # Remove RAFT from memory + RAFT_clear_memory() + sdcn_anim_tmp.frames_prepared = True + else: + # process frame + sdcn_anim_tmp.frames_prepared = False + + cn = sdcn_anim_tmp.process_counter % 10 + curr_frame = sdcn_anim_tmp.prepared_frames[cn+1][...,:3] + prev_frame = sdcn_anim_tmp.prepared_frames[cn][...,:3] + next_flow = sdcn_anim_tmp.prepared_next_flows[cn] + prev_flow = sdcn_anim_tmp.prepared_prev_flows[cn] + + ### STEP 1 + alpha_mask, warped_styled_frame = compute_diff_map(next_flow, prev_flow, prev_frame, curr_frame, sdcn_anim_tmp.prev_frame_styled, args_dict) + warped_styled_frame_ = warped_styled_frame.copy() + + #fl_w, fl_h = prev_flow.shape[:2] + #prev_flow_n = prev_flow / np.array([fl_h,fl_w]) + #flow_mask = np.clip(1 - np.linalg.norm(prev_flow_n, axis=-1)[...,None] * 20, 0, 1) + #alpha_mask = alpha_mask * flow_mask + + if sdcn_anim_tmp.process_counter > 0 and args_dict['occlusion_mask_trailing']: + alpha_mask = alpha_mask + sdcn_anim_tmp.prev_frame_alpha_mask * 0.5 + sdcn_anim_tmp.prev_frame_alpha_mask = alpha_mask + + # alpha_mask = np.round(alpha_mask * 8) / 8 #> 0.3 + alpha_mask = np.clip(alpha_mask, 0, 1) + occlusion_mask = np.clip(alpha_mask * 255, 0, 255).astype(np.uint8) + + # fix warped styled frame from duplicated that occures on the places where flow is zero, but only because there is no place to get the color from + warped_styled_frame = curr_frame.astype(float) * alpha_mask + warped_styled_frame.astype(float) * (1 - alpha_mask) + + # process current frame + # TODO: convert args_dict into separate dict that stores only params necessery for img2img processing + img2img_args_dict = args_dict #copy.deepcopy(args_dict) + img2img_args_dict['denoising_strength'] = args_dict['processing_strength'] + if args_dict['step_1_processing_mode'] == 0: # Process full image then blend in occlusions + img2img_args_dict['mode'] = 0 + img2img_args_dict['mask_img'] = None #Image.fromarray(occlusion_mask) + elif args_dict['step_1_processing_mode'] == 1: # Inpaint occlusions + img2img_args_dict['mode'] = 4 + img2img_args_dict['mask_img'] = Image.fromarray(occlusion_mask) + else: + raise Exception('Incorrect step 1 processing mode!') + + blend_alpha = args_dict['step_1_blend_alpha'] + init_img = warped_styled_frame * (1 - blend_alpha) + curr_frame * blend_alpha + img2img_args_dict['init_img'] = Image.fromarray(np.clip(init_img, 0, 255).astype(np.uint8)) + img2img_args_dict['seed'] = args_dict['step_1_seed'] + utils.set_CNs_input_image(img2img_args_dict, Image.fromarray(curr_frame)) + processed_frames, _, _, _ = utils.img2img(img2img_args_dict) + processed_frame = np.array(processed_frames[0])[...,:3] + + # normalizing the colors + processed_frame = skimage.exposure.match_histograms(processed_frame, curr_frame, channel_axis=None) + processed_frame = processed_frame.astype(float) * alpha_mask + warped_styled_frame.astype(float) * (1 - alpha_mask) + + #processed_frame = processed_frame * 0.94 + curr_frame * 0.06 + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + sdcn_anim_tmp.prev_frame_styled = processed_frame.copy() + + ### STEP 2 + if args_dict['fix_frame_strength'] > 0: + img2img_args_dict = args_dict #copy.deepcopy(args_dict) + img2img_args_dict['mode'] = 0 + img2img_args_dict['init_img'] = Image.fromarray(processed_frame) + img2img_args_dict['mask_img'] = None + img2img_args_dict['denoising_strength'] = args_dict['fix_frame_strength'] + img2img_args_dict['seed'] = args_dict['step_2_seed'] + utils.set_CNs_input_image(img2img_args_dict, Image.fromarray(curr_frame)) + processed_frames, _, _, _ = utils.img2img(img2img_args_dict) + processed_frame = np.array(processed_frames[0]) + processed_frame = skimage.exposure.match_histograms(processed_frame, curr_frame, channel_axis=None) + + processed_frame = np.clip(processed_frame, 0, 255).astype(np.uint8) + warped_styled_frame_ = np.clip(warped_styled_frame_, 0, 255).astype(np.uint8) + + # Write the frame to the output video + frame_out = np.clip(processed_frame, 0, 255).astype(np.uint8) + frame_out = cv2.cvtColor(frame_out, cv2.COLOR_RGB2BGR) + sdcn_anim_tmp.output_video.write(frame_out) + + sdcn_anim_tmp.process_counter += 1 + #if sdcn_anim_tmp.process_counter >= sdcn_anim_tmp.total_frames - 1: + # sdcn_anim_tmp.input_video.release() + # sdcn_anim_tmp.output_video.release() + # sdcn_anim_tmp.prev_frame = None + + save_result_to_image(processed_frame, sdcn_anim_tmp.process_counter + 1) + + stat = get_cur_stat() + utils.get_time_left(step+2, loop_iterations+1, processing_start_time) + yield stat, curr_frame, occlusion_mask, warped_styled_frame_, processed_frame, None, gr.Button.update(interactive=False), gr.Button.update(interactive=True) + + RAFT_clear_memory() + + sdcn_anim_tmp.input_video.release() + sdcn_anim_tmp.output_video.release() + + curr_frame = gr.Image.update() + occlusion_mask = gr.Image.update() + warped_styled_frame_ = gr.Image.update() + processed_frame = gr.Image.update() + + yield get_cur_stat(), curr_frame, occlusion_mask, warped_styled_frame_, processed_frame, output_video_name, gr.Button.update(interactive=True), gr.Button.update(interactive=False) \ No newline at end of file diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/__pycache__/civitai_helper.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/__pycache__/civitai_helper.cpython-310.pyc index 8316470304baee4ca3aa0d394a7b7aaecd8c5ce9..c03a5c92e2729d072a7ae0fe128e02c0dbb01459 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/__pycache__/civitai_helper.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/__pycache__/civitai_helper.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/__init__.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/__init__.cpython-310.pyc index cfb6b12c7083a8d9d01adc8682fe32b0365f9ee8..07ec1abe650a43258f58c2ae3707e3a46f9e9240 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/__init__.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/__init__.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/civitai.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/civitai.cpython-310.pyc index 13995b18a1267d7658f2810ab786239d7a9a0adc..2974b8625bc653b45653f9b992f3743019725583 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/civitai.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/civitai.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/downloader.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/downloader.cpython-310.pyc index 15e341adcbe9817d29487951bbac3b6b50c05b4e..39b5640c2db7d56212d8987c33a1e8aa5bf1fa1a 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/downloader.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/downloader.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/js_action_civitai.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/js_action_civitai.cpython-310.pyc index d3b92da0f0bf94fa7501346089c0b608bbf20fd7..10d1d6220272dd54fc412947ab9f16e5a6aa953b 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/js_action_civitai.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/js_action_civitai.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model.cpython-310.pyc index 64a8bc356d1b637591db984eedb23a56f283be1a..2da86538961480f3f82db218731fe75e31b4dbeb 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model_action_civitai.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model_action_civitai.cpython-310.pyc index 585e89c9766512532f6e3b97f3769699ae53e96c..874f44b246a3e07a7c86779f7723e6ccc164a04a 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model_action_civitai.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/model_action_civitai.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/msg_handler.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/msg_handler.cpython-310.pyc index f60d0880a4a335b82a468046462da3b5d337ab9d..d66e8ec9a7a578fcb2c7ea5be5067c545816b1a6 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/msg_handler.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/msg_handler.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/setting.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/setting.cpython-310.pyc index aeffd25b4ff90c26e1a8c03d83a9fcc8a5bf0bb0..0ff628599d4395ce388fab1fa2d3317b971ad69d 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/setting.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/setting.cpython-310.pyc differ diff --git a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/util.cpython-310.pyc b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/util.cpython-310.pyc index b72479dcd78a5f9ea5f0d7095e5ada5bb71c4f46..5d293ef350dca51fdeb9c1bc64b935f54b6d85c6 100644 Binary files a/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/util.cpython-310.pyc and b/Stable-Diffusion-Webui-Civitai-Helper/scripts/ch_lib/__pycache__/util.cpython-310.pyc differ diff --git a/a1111-microsoftexcel-locon/__pycache__/locon.cpython-310.pyc b/a1111-microsoftexcel-locon/__pycache__/locon.cpython-310.pyc index a525aa3417009759447129e49b24c25865156fa2..ea30789fd61784290a62a26e3d46f82966d1d23f 100644 Binary files a/a1111-microsoftexcel-locon/__pycache__/locon.cpython-310.pyc and b/a1111-microsoftexcel-locon/__pycache__/locon.cpython-310.pyc differ diff --git a/a1111-microsoftexcel-locon/__pycache__/locon_compvis.cpython-310.pyc b/a1111-microsoftexcel-locon/__pycache__/locon_compvis.cpython-310.pyc index d53f1e4e8ccdad42cb73b6f2f383f0f3597bfaa0..e4f2f656fd6cf619ea9be07c424cc23a3b531d6c 100644 Binary files a/a1111-microsoftexcel-locon/__pycache__/locon_compvis.cpython-310.pyc and b/a1111-microsoftexcel-locon/__pycache__/locon_compvis.cpython-310.pyc differ diff --git a/a1111-microsoftexcel-locon/scripts/__pycache__/main.cpython-310.pyc b/a1111-microsoftexcel-locon/scripts/__pycache__/main.cpython-310.pyc index 9e5059aaaafcd822e18eda5213ecb27ffcacf0a7..a35ac65e207266dcc44de55cbdeb1102c8e5bf7f 100644 Binary files a/a1111-microsoftexcel-locon/scripts/__pycache__/main.cpython-310.pyc and b/a1111-microsoftexcel-locon/scripts/__pycache__/main.cpython-310.pyc differ diff --git a/a1111-microsoftexcel-tagcomplete/scripts/__pycache__/tag_autocomplete_helper.cpython-310.pyc b/a1111-microsoftexcel-tagcomplete/scripts/__pycache__/tag_autocomplete_helper.cpython-310.pyc index b24ef70aad6ff07ad5f0bb7cf0dc4f272b6d66bc..76d7b78a6b2191b7cc035ce74df54ca174bc46ca 100644 Binary files a/a1111-microsoftexcel-tagcomplete/scripts/__pycache__/tag_autocomplete_helper.cpython-310.pyc and b/a1111-microsoftexcel-tagcomplete/scripts/__pycache__/tag_autocomplete_helper.cpython-310.pyc differ diff --git a/a1111-sd-webui-lycoris/__pycache__/extra_networks_lyco.cpython-310.pyc b/a1111-sd-webui-lycoris/__pycache__/extra_networks_lyco.cpython-310.pyc index 335cfe421ff99914b105fb5687c935d0dfe25022..478097c25a01340e7338f2a0b01949a63dfd6b40 100644 Binary files a/a1111-sd-webui-lycoris/__pycache__/extra_networks_lyco.cpython-310.pyc and b/a1111-sd-webui-lycoris/__pycache__/extra_networks_lyco.cpython-310.pyc differ diff --git a/a1111-sd-webui-lycoris/__pycache__/lycoris.cpython-310.pyc b/a1111-sd-webui-lycoris/__pycache__/lycoris.cpython-310.pyc index 382e92a335e1a78e7e8983384283381232672c80..076a3cde6c2c9bacd56454bbc5af118ead15b4a8 100644 Binary files a/a1111-sd-webui-lycoris/__pycache__/lycoris.cpython-310.pyc and b/a1111-sd-webui-lycoris/__pycache__/lycoris.cpython-310.pyc differ diff --git a/a1111-sd-webui-lycoris/__pycache__/preload.cpython-310.pyc b/a1111-sd-webui-lycoris/__pycache__/preload.cpython-310.pyc index 1e87f39a71c464810722cdfd8ca9287e06b92408..881ca86afb24804be4e01d42e41698b03a61c1cb 100644 Binary files a/a1111-sd-webui-lycoris/__pycache__/preload.cpython-310.pyc and b/a1111-sd-webui-lycoris/__pycache__/preload.cpython-310.pyc differ diff --git a/a1111-sd-webui-lycoris/__pycache__/ui_extra_networks_lyco.cpython-310.pyc b/a1111-sd-webui-lycoris/__pycache__/ui_extra_networks_lyco.cpython-310.pyc index 41cf3b3d9a281e5518d9d79d9506d1808ba029f7..856f48497d2952e6d145690bbb314ccc964466c4 100644 Binary files a/a1111-sd-webui-lycoris/__pycache__/ui_extra_networks_lyco.cpython-310.pyc and b/a1111-sd-webui-lycoris/__pycache__/ui_extra_networks_lyco.cpython-310.pyc differ diff --git a/a1111-sd-webui-lycoris/scripts/__pycache__/lycoris_script.cpython-310.pyc b/a1111-sd-webui-lycoris/scripts/__pycache__/lycoris_script.cpython-310.pyc index 663b7e41e345b50fcc720e984936dcef3e8b8c66..16f292b9b1db5cce2dc7af50ab213c84582a0b34 100644 Binary files a/a1111-sd-webui-lycoris/scripts/__pycache__/lycoris_script.cpython-310.pyc and b/a1111-sd-webui-lycoris/scripts/__pycache__/lycoris_script.cpython-310.pyc differ diff --git a/addtional/__pycache__/preload.cpython-310.pyc b/addtional/__pycache__/preload.cpython-310.pyc index 78c6e363e56144aaf1da1fa522533282ac4146b5..675977c28efe739c08ae2ed58e67973230fa211c 100644 Binary files a/addtional/__pycache__/preload.cpython-310.pyc and b/addtional/__pycache__/preload.cpython-310.pyc differ diff --git a/addtional/hashes.json b/addtional/hashes.json index f82170446ed4eb3357922cde1f9419f5803543d9..94e7e79bbe0d19e60b9eb029458405bcc00c87d2 100644 --- a/addtional/hashes.json +++ b/addtional/hashes.json @@ -3,32 +3,37 @@ "/content/microsoftexcel/models/Lora/GoodHands-beta2.safetensors": { "model": "c05ed279295ed06ad8c459254bd9d910d46d1dab4de5dbdb59e9d5a231b8b43f", "legacy": "5275bd79", - "mtime": 1685856032.911875 + "mtime": 1687338891.6372578 }, "/content/microsoftexcel/models/Lora/Caitlyn.safetensors": { "model": "fe2d779164cb23b97a10b926d809fc94d074e1e7deb595be615b8ad11ed5fd11", "legacy": "40fa34e7", - "mtime": 1685856018.1636622 + "mtime": 1687338884.7806156 }, "/content/microsoftexcel/models/Lora/ireliav2-000034.safetensors": { "model": "7fd4ca5cdc40f1d61548e39149b9c0eff6bac17601469abf7de72a42beea245d", "legacy": "d7774c83", - "mtime": 1685856017.3855984 + "mtime": 1687338883.9015331 }, "/content/microsoftexcel/models/Lora/ratatatat74-000030.safetensors": { "model": "43f33c0dd794e5ae4cb4cd1d12d3ad70466c61084c43bf105750790504dc8bdd", "legacy": "e5787969", - "mtime": 1685856021.2959197 + "mtime": 1687338886.6787932 }, "/content/microsoftexcel/models/Lora/nixeu_offset64dim.safetensors": { "model": "3bfe61233b402e3663c136396c3bdea4cfd982215429e9ee83bff505692e6bbe", "legacy": "86bd75dd", - "mtime": 1685856013.7583003 + "mtime": 1687338882.9234416 }, "/content/microsoftexcel/models/Lora/sukuna.safetensors": { "model": "89a59997da996808d64cecae57b23cb788118cfc3c02815faf7a1857c4babcee", "legacy": "f2ac6ed8", - "mtime": 1685856541.630725 + "mtime": 1687338879.7851477 + }, + "/content/microsoftexcel/models/Lora/MakimaCSM.safetensors": { + "model": "0b7b2682d8a6c06208d62aab8534af8e1251dbc2d8a66962df80455a8083fad9", + "legacy": "11d289ef", + "mtime": 1687344071.8123095 } } } \ No newline at end of file diff --git a/addtional/scripts/__pycache__/additional_networks.cpython-310.pyc b/addtional/scripts/__pycache__/additional_networks.cpython-310.pyc index a3c0f7f3f420b8fcd9ea54d4453f27ccf626d054..5e1f4cea4e6914dd31aba6c5464b0dc8bcbf5d0e 100644 Binary files a/addtional/scripts/__pycache__/additional_networks.cpython-310.pyc and b/addtional/scripts/__pycache__/additional_networks.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/addnet_xyz_grid_support.cpython-310.pyc b/addtional/scripts/__pycache__/addnet_xyz_grid_support.cpython-310.pyc index d81f4b307fadc82ef5e89515b7c3ecad0322da73..a4a78298c01a4c51a2754e0949ca75dc6375238a 100644 Binary files a/addtional/scripts/__pycache__/addnet_xyz_grid_support.cpython-310.pyc and b/addtional/scripts/__pycache__/addnet_xyz_grid_support.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/lora_compvis.cpython-310.pyc b/addtional/scripts/__pycache__/lora_compvis.cpython-310.pyc index 0dbbc5b19332d9a21baba934aa9ec805c643246b..c1513f3ecc75489ffbc310e586b2977f38e2c161 100644 Binary files a/addtional/scripts/__pycache__/lora_compvis.cpython-310.pyc and b/addtional/scripts/__pycache__/lora_compvis.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/metadata_editor.cpython-310.pyc b/addtional/scripts/__pycache__/metadata_editor.cpython-310.pyc index 21579db8d4257223ec43d1adb31f6438cad3eb8d..3409a98a05e5de080816fe71be266dea0c3b4eb9 100644 Binary files a/addtional/scripts/__pycache__/metadata_editor.cpython-310.pyc and b/addtional/scripts/__pycache__/metadata_editor.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/model_util.cpython-310.pyc b/addtional/scripts/__pycache__/model_util.cpython-310.pyc index 00d97a439c62b045a56ab4f78aaad066f80fb918..d342f679e9074fa7ae30bf0e3a72a2cbba367190 100644 Binary files a/addtional/scripts/__pycache__/model_util.cpython-310.pyc and b/addtional/scripts/__pycache__/model_util.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/safetensors_hack.cpython-310.pyc b/addtional/scripts/__pycache__/safetensors_hack.cpython-310.pyc index cfcaab8342ca911eb6cf9f1a0e7351f0ef184d37..08ff26b7b5e1fe3cfcef03db80f46115bee6fb89 100644 Binary files a/addtional/scripts/__pycache__/safetensors_hack.cpython-310.pyc and b/addtional/scripts/__pycache__/safetensors_hack.cpython-310.pyc differ diff --git a/addtional/scripts/__pycache__/util.cpython-310.pyc b/addtional/scripts/__pycache__/util.cpython-310.pyc index 05b6642240f5992e1d876abbffc973b9f4553bad..40e5072d1c40d0c4892c7a323c4609ebefe56a5f 100644 Binary files a/addtional/scripts/__pycache__/util.cpython-310.pyc and b/addtional/scripts/__pycache__/util.cpython-310.pyc differ diff --git a/adetailer/__pycache__/preload.cpython-310.pyc b/adetailer/__pycache__/preload.cpython-310.pyc index 201f39ea9460190cb6ce6bdc69b4f601986412ea..ee25d5a0c988c8e2699fa2aef46383d52e889722 100644 Binary files a/adetailer/__pycache__/preload.cpython-310.pyc and b/adetailer/__pycache__/preload.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/__init__.cpython-310.pyc b/adetailer/adetailer/__pycache__/__init__.cpython-310.pyc index 586a5a8bdb533733458914f4632509a490cba63c..8afa6c64bfbbbf7df54981a65db1c1aa44bfe12d 100644 Binary files a/adetailer/adetailer/__pycache__/__init__.cpython-310.pyc and b/adetailer/adetailer/__pycache__/__init__.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/__version__.cpython-310.pyc b/adetailer/adetailer/__pycache__/__version__.cpython-310.pyc index 8e1785be7a6805751af401e2984aa0d91825650c..73a4628f80b2e26bd519907d0ea48204b93ebd8e 100644 Binary files a/adetailer/adetailer/__pycache__/__version__.cpython-310.pyc and b/adetailer/adetailer/__pycache__/__version__.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/args.cpython-310.pyc b/adetailer/adetailer/__pycache__/args.cpython-310.pyc index 035dbc3d6274d3164d1d83a482adfe476eaa0e2f..caad26b4ccf7a0a0e0ed8c3f7b55e4259a6aaab5 100644 Binary files a/adetailer/adetailer/__pycache__/args.cpython-310.pyc and b/adetailer/adetailer/__pycache__/args.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/common.cpython-310.pyc b/adetailer/adetailer/__pycache__/common.cpython-310.pyc index da551e4891edfc020ba6565efad0baa4f54eb7f2..ad5190aab4a8e1aac3bd1a7a538d19b329f47b25 100644 Binary files a/adetailer/adetailer/__pycache__/common.cpython-310.pyc and b/adetailer/adetailer/__pycache__/common.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/mask.cpython-310.pyc b/adetailer/adetailer/__pycache__/mask.cpython-310.pyc index f231ab7f8a750b7720a702dc1181b8eb643daeba..14ba8fabfc945971674902395035465bb213309d 100644 Binary files a/adetailer/adetailer/__pycache__/mask.cpython-310.pyc and b/adetailer/adetailer/__pycache__/mask.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/mediapipe.cpython-310.pyc b/adetailer/adetailer/__pycache__/mediapipe.cpython-310.pyc index 1c1b30f428eafc048d01b47082971de00c00e5db..df5ad8304b49f8dd47892f77b0be85b582e35387 100644 Binary files a/adetailer/adetailer/__pycache__/mediapipe.cpython-310.pyc and b/adetailer/adetailer/__pycache__/mediapipe.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/ui.cpython-310.pyc b/adetailer/adetailer/__pycache__/ui.cpython-310.pyc index c9965d9985b216885285561cb5079bd60140cf17..09fc685b8858610c6cbd4d7d64a5b6f0c79f6bc3 100644 Binary files a/adetailer/adetailer/__pycache__/ui.cpython-310.pyc and b/adetailer/adetailer/__pycache__/ui.cpython-310.pyc differ diff --git a/adetailer/adetailer/__pycache__/ultralytics.cpython-310.pyc b/adetailer/adetailer/__pycache__/ultralytics.cpython-310.pyc index 749de14d2181eeb90d38e1e0bae57ada37a99ab3..fffc32db43af9feb153cfbb4dc2448471e287704 100644 Binary files a/adetailer/adetailer/__pycache__/ultralytics.cpython-310.pyc and b/adetailer/adetailer/__pycache__/ultralytics.cpython-310.pyc differ diff --git a/adetailer/controlnet_ext/__pycache__/__init__.cpython-310.pyc b/adetailer/controlnet_ext/__pycache__/__init__.cpython-310.pyc index 5acb26b5f0103152d114e0bdf64444ad224fedbd..0df262d94a20fc4fccae1437b0015b0cbb85de65 100644 Binary files a/adetailer/controlnet_ext/__pycache__/__init__.cpython-310.pyc and b/adetailer/controlnet_ext/__pycache__/__init__.cpython-310.pyc differ diff --git a/adetailer/controlnet_ext/__pycache__/controlnet_ext.cpython-310.pyc b/adetailer/controlnet_ext/__pycache__/controlnet_ext.cpython-310.pyc index 71b4d0037cf694aec32262790ca62bec70b1a454..c4204b2ab53ab011113601f9a17c9842d44a08d1 100644 Binary files a/adetailer/controlnet_ext/__pycache__/controlnet_ext.cpython-310.pyc and b/adetailer/controlnet_ext/__pycache__/controlnet_ext.cpython-310.pyc differ diff --git a/adetailer/controlnet_ext/__pycache__/restore.cpython-310.pyc b/adetailer/controlnet_ext/__pycache__/restore.cpython-310.pyc index 36b9ad7df387cb174ccd746cec7aa9e82bc8f04d..5a9ba7be1670c9223581c667123decc5ec0987ec 100644 Binary files a/adetailer/controlnet_ext/__pycache__/restore.cpython-310.pyc and b/adetailer/controlnet_ext/__pycache__/restore.cpython-310.pyc differ diff --git a/adetailer/scripts/__pycache__/!adetailer.cpython-310.pyc b/adetailer/scripts/__pycache__/!adetailer.cpython-310.pyc index 48235e68602de7d421e6e6377f635fa620235e8d..effc23b6fef81cdb0ac920e6d4eb5cc99435b728 100644 Binary files a/adetailer/scripts/__pycache__/!adetailer.cpython-310.pyc and b/adetailer/scripts/__pycache__/!adetailer.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/__init__.cpython-310.pyc b/adetailer/sd_webui/__pycache__/__init__.cpython-310.pyc index 8fe564e90c48b4bc687b6bb555ae15744302aed0..549c360e20c61dd371f70b7cd5eafde70edca30a 100644 Binary files a/adetailer/sd_webui/__pycache__/__init__.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/__init__.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/images.cpython-310.pyc b/adetailer/sd_webui/__pycache__/images.cpython-310.pyc index 38962c75e1944af09707b376436263427d270029..da639cd7ce3917fdaff44ec950474f8002231497 100644 Binary files a/adetailer/sd_webui/__pycache__/images.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/images.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/paths.cpython-310.pyc b/adetailer/sd_webui/__pycache__/paths.cpython-310.pyc index 60bb573d89acfabef4a611df142389a40d8c0dea..3548ab37c35478a2166214ed5f06ae87901be3ab 100644 Binary files a/adetailer/sd_webui/__pycache__/paths.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/paths.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/processing.cpython-310.pyc b/adetailer/sd_webui/__pycache__/processing.cpython-310.pyc index ab5e9f8d8ba9553f507785d9c2dd94e6f4f94385..6b6d06a6777ba8711fb8c77fcb0ccdf465477b74 100644 Binary files a/adetailer/sd_webui/__pycache__/processing.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/processing.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/safe.cpython-310.pyc b/adetailer/sd_webui/__pycache__/safe.cpython-310.pyc index a0aca9eababe5bf28b3aa70ff7851932a7ab7063..cf2ef8726c3edaabb49f6324da26ea796bf6358a 100644 Binary files a/adetailer/sd_webui/__pycache__/safe.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/safe.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/script_callbacks.cpython-310.pyc b/adetailer/sd_webui/__pycache__/script_callbacks.cpython-310.pyc index 7b6b2a0d208dae67c7766d82742b5d0a0304b847..100b6cf9a43e5289c89c04da72aac29f8eadd18a 100644 Binary files a/adetailer/sd_webui/__pycache__/script_callbacks.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/script_callbacks.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/scripts.cpython-310.pyc b/adetailer/sd_webui/__pycache__/scripts.cpython-310.pyc index 50cb4acfcde342b3e477d91c927fd05e3062fbf9..9c335331d1e172d1006afe506e869b7d2db9e459 100644 Binary files a/adetailer/sd_webui/__pycache__/scripts.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/scripts.cpython-310.pyc differ diff --git a/adetailer/sd_webui/__pycache__/shared.cpython-310.pyc b/adetailer/sd_webui/__pycache__/shared.cpython-310.pyc index 51e845c10f899c37257b9c8b2be0d9a35b5d4a62..41d8a95682acdffec0121eb56fdb30762bf03136 100644 Binary files a/adetailer/sd_webui/__pycache__/shared.cpython-310.pyc and b/adetailer/sd_webui/__pycache__/shared.cpython-310.pyc differ diff --git a/ebsynth_utility/README.md b/ebsynth_utility/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ac30c03427d56ba94edc51337247904487162d87 --- /dev/null +++ b/ebsynth_utility/README.md @@ -0,0 +1,186 @@ +# ebsynth_utility + +## Overview +#### AUTOMATIC1111 UI extension for creating videos using img2img and ebsynth. +#### This extension allows you to output edited videos using ebsynth.(AE is not required) + + +##### With [Controlnet](https://github.com/Mikubill/sd-webui-controlnet) installed, I have confirmed that all features of this extension are working properly! +##### [Controlnet](https://github.com/Mikubill/sd-webui-controlnet) is a must for video editing, so I recommend installing it. +##### Multi ControlNet("canny" + "normal map") would be suitable for video editing. + +
+ +###### I made a new extension to make videos in a different way than this extension. You can use it if you like. +###### [sd_loopback_music_sync_wave](https://github.com/s9roll7/sd_loopback_music_sync_wave) + + +
+ + +## Example +- The following sample is raw output of this extension. +#### sample 1 mask with [clipseg](https://github.com/timojl/clipseg) +- first from left : original +- second from left : masking "cat" exclude "finger" +- third from left : masking "cat head" +- right : color corrected with [color-matcher](https://github.com/hahnec/color-matcher) (see stage 3.5) +- Multiple targets can also be specified.(e.g. cat,dog,boy,girl) + + +#### sample 2 blend background +- person : masterpiece, best quality, masterpiece, 1girl, masterpiece, best quality,anime screencap, anime style +- background : cyberpunk, factory, room ,anime screencap, anime style +- It is also possible to blend with your favorite videos. + + +#### sample 3 auto tagging +- left : original +- center : apply the same prompts in all keyframes +- right : apply auto tagging by deepdanbooru in all keyframes +- This function improves the detailed changes in facial expressions, hand expressions, etc. + In the sample video, the "closed_eyes" and "hands_on_own_face" tags have been added to better represent eye blinks and hands brought in front of the face. + + +#### sample 4 auto tagging (apply lora dynamically) +- left : apply auto tagging by deepdanbooru in all keyframes +- right : apply auto tagging by deepdanbooru in all keyframes + apply "anyahehface" lora dynamically +- Added the function to dynamically apply TI, hypernet, Lora, and additional prompts according to automatically attached tags. + In the sample video, if the "smile" tag is given, the lora and lora trigger keywords are set to be added according to the strength of the "smile" tag. + Also, since automatically added tags are sometimes incorrect, unnecessary tags are listed in the blacklist. + [Here](sample/) is the actual configuration file used. placed in "Project directory" for use. + + +
+ +## Installation +- Install [ffmpeg](https://ffmpeg.org/) for your operating system + (https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/) +- Install [Ebsynth](https://ebsynth.com/) +- Use the Extensions tab of the webui to [Install from URL] + +
+
+ +## Usage +- Go to [Ebsynth Utility] tab. +- Create an empty directory somewhere, and fill in the "Project directory" field. +- Place the video you want to edit from somewhere, and fill in the "Original Movie Path" field. + Use short videos of a few seconds at first. +- Select stage 1 and Generate. +- Execute in order from stage 1 to 7. + Progress during the process is not reflected in webui, so please check the console screen. + If you see "completed." in webui, it is completed. +(In the current latest webui, it seems to cause an error if you do not drop the image on the main screen of img2img. +Please drop the image as it does not affect the result.) + +
+
+ +## Note 1 +For reference, here's what I did when I edited a 1280x720 30fps 15sec video based on +#### Stage 1 +There is nothing to configure. +All frames of the video and mask images for all frames are generated. + +#### Stage 2 +In the implementation of this extension, the keyframe interval is chosen to be shorter where there is a lot of motion and longer where there is little motion. +If the animation breaks up, increase the keyframe, if it flickers, decrease the keyframe. +First, generate one time with the default settings and go straight ahead without worrying about the result. + + +#### Stage 3 +Select one of the keyframes, throw it to img2img, and run [Interrogate DeepBooru]. +Delete unwanted words such as blur from the displayed prompt. +Fill in the rest of the settings as you would normally do for image generation. + +Here is the settings I used. +- Sampling method : Euler a +- Sampling Steps : 50 +- Width : 960 +- Height : 512 +- CFG Scale : 20 +- Denoising strength : 0.2 + +Here is the settings for extension. +- Mask Mode(Override img2img Mask mode) : Normal +- Img2Img Repeat Count (Loop Back) : 5 +- Add N to seed when repeating : 1 +- use Face Crop img2img : True +- Face Detection Method : YuNet +- Max Crop Size : 1024 +- Face Denoising Strength : 0.25 +- Face Area Magnification : 1.5 (The larger the number, the closer to the model's painting style, but the more likely it is to shift when merged with the body.) +- Enable Face Prompt : False + +Trial and error in this process is the most time-consuming part. +Monitor the destination folder and if you do not like results, interrupt and change the settings. +[Prompt][Denoising strength] and [Face Denoising Strength] settings when using Face Crop img2img will greatly affect the result. +For more information on Face Crop img2img, check [here](https://github.com/s9roll7/face_crop_img2img) + +If you have lots of memory to spare, increasing the width and height values while maintaining the aspect ratio may greatly improve results. + +This extension may help with the adjustment. +https://github.com/s9roll7/img2img_for_all_method + +
+ +**The information above is from a time when there was no controlnet. +When controlnet are used together (especially multi-controlnets), +Even setting "Denoising strength" to a high value works well, and even setting it to 1.0 produces meaningful results. +If "Denoising strength" is set to a high value, "Loop Back" can be set to 1.** + +
+ +#### Stage 4 +Scale it up or down and process it to exactly the same size as the original video. +This process should only need to be done once. + +- Width : 1280 +- Height : 720 +- Upscaler 1 : R-ESRGAN 4x+ +- Upscaler 2 : R-ESRGAN 4x+ Anime6B +- Upscaler 2 visibility : 0.5 +- GFPGAN visibility : 1 +- CodeFormer visibility : 0 +- CodeFormer weight : 0 + +#### Stage 5 +There is nothing to configure. +.ebs file will be generated. + +#### Stage 6 +Run the .ebs file. +I wouldn't change the settings, but you could adjust the .ebs settings. + +#### Stage 7 +Finally, output the video. +In my case, the entire process from 1 to 7 took about 30 minutes. + +- Crossfade blend rate : 1.0 +- Export type : mp4 + +
+
+ +## Note 2 : How to use multi-controlnet together +#### in webui setting + +
+#### In controlnet settings in img2img tab(for controlnet 0) + +
+#### In controlnet settings in img2img tab(for controlnet 1) + +
+#### In ebsynth_utility settings in img2img tab +**Warning : "Weight" in the controlnet settings is overridden by the following values** + + +
+
+ +## Note 3 : How to use clipseg + + + diff --git a/ebsynth_utility/__pycache__/calculator.cpython-310.pyc b/ebsynth_utility/__pycache__/calculator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8788e77987e2f9a1299209c776354e1a78a87946 Binary files /dev/null and b/ebsynth_utility/__pycache__/calculator.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/ebsynth_utility.cpython-310.pyc b/ebsynth_utility/__pycache__/ebsynth_utility.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50bf5e597ab311617bbfc65b60399dc2d98237aa Binary files /dev/null and b/ebsynth_utility/__pycache__/ebsynth_utility.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage1.cpython-310.pyc b/ebsynth_utility/__pycache__/stage1.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e1e9adf226c3a7911d248e6d0c9c1a15be6b33b1 Binary files /dev/null and b/ebsynth_utility/__pycache__/stage1.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage2.cpython-310.pyc b/ebsynth_utility/__pycache__/stage2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e88f3a62e6b48ed7f296ac950c5bcb64e3af66d Binary files /dev/null and b/ebsynth_utility/__pycache__/stage2.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage3_5.cpython-310.pyc b/ebsynth_utility/__pycache__/stage3_5.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1aaaef3a875b073ed072f9b9169767120d19c6f2 Binary files /dev/null and b/ebsynth_utility/__pycache__/stage3_5.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage5.cpython-310.pyc b/ebsynth_utility/__pycache__/stage5.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e3e09360fb6dd6c8c773cb6bfd304d051b5a9c5 Binary files /dev/null and b/ebsynth_utility/__pycache__/stage5.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage7.cpython-310.pyc b/ebsynth_utility/__pycache__/stage7.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..26bde6ff258cff9f5d62d2de8c94a0455efb6d84 Binary files /dev/null and b/ebsynth_utility/__pycache__/stage7.cpython-310.pyc differ diff --git a/ebsynth_utility/__pycache__/stage8.cpython-310.pyc b/ebsynth_utility/__pycache__/stage8.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68c10f76529cd81c68d422ddaa1961f30023c70f Binary files /dev/null and b/ebsynth_utility/__pycache__/stage8.cpython-310.pyc differ diff --git a/ebsynth_utility/calculator.py b/ebsynth_utility/calculator.py new file mode 100644 index 0000000000000000000000000000000000000000..35d66bfe960d86b0a56ce207cca3725f303a2da9 --- /dev/null +++ b/ebsynth_utility/calculator.py @@ -0,0 +1,237 @@ +# https://www.mycompiler.io/view/3TFZagC + +class ParseError(Exception): + def __init__(self, pos, msg, *args): + self.pos = pos + self.msg = msg + self.args = args + + def __str__(self): + return '%s at position %s' % (self.msg % self.args, self.pos) + +class Parser: + def __init__(self): + self.cache = {} + + def parse(self, text): + self.text = text + self.pos = -1 + self.len = len(text) - 1 + rv = self.start() + self.assert_end() + return rv + + def assert_end(self): + if self.pos < self.len: + raise ParseError( + self.pos + 1, + 'Expected end of string but got %s', + self.text[self.pos + 1] + ) + + def eat_whitespace(self): + while self.pos < self.len and self.text[self.pos + 1] in " \f\v\r\t\n": + self.pos += 1 + + def split_char_ranges(self, chars): + try: + return self.cache[chars] + except KeyError: + pass + + rv = [] + index = 0 + length = len(chars) + + while index < length: + if index + 2 < length and chars[index + 1] == '-': + if chars[index] >= chars[index + 2]: + raise ValueError('Bad character range') + + rv.append(chars[index:index + 3]) + index += 3 + else: + rv.append(chars[index]) + index += 1 + + self.cache[chars] = rv + return rv + + def char(self, chars=None): + if self.pos >= self.len: + raise ParseError( + self.pos + 1, + 'Expected %s but got end of string', + 'character' if chars is None else '[%s]' % chars + ) + + next_char = self.text[self.pos + 1] + if chars == None: + self.pos += 1 + return next_char + + for char_range in self.split_char_ranges(chars): + if len(char_range) == 1: + if next_char == char_range: + self.pos += 1 + return next_char + elif char_range[0] <= next_char <= char_range[2]: + self.pos += 1 + return next_char + + raise ParseError( + self.pos + 1, + 'Expected %s but got %s', + 'character' if chars is None else '[%s]' % chars, + next_char + ) + + def keyword(self, *keywords): + self.eat_whitespace() + if self.pos >= self.len: + raise ParseError( + self.pos + 1, + 'Expected %s but got end of string', + ','.join(keywords) + ) + + for keyword in keywords: + low = self.pos + 1 + high = low + len(keyword) + + if self.text[low:high] == keyword: + self.pos += len(keyword) + self.eat_whitespace() + return keyword + + raise ParseError( + self.pos + 1, + 'Expected %s but got %s', + ','.join(keywords), + self.text[self.pos + 1], + ) + + def match(self, *rules): + self.eat_whitespace() + last_error_pos = -1 + last_exception = None + last_error_rules = [] + + for rule in rules: + initial_pos = self.pos + try: + rv = getattr(self, rule)() + self.eat_whitespace() + return rv + except ParseError as e: + self.pos = initial_pos + + if e.pos > last_error_pos: + last_exception = e + last_error_pos = e.pos + last_error_rules.clear() + last_error_rules.append(rule) + elif e.pos == last_error_pos: + last_error_rules.append(rule) + + if len(last_error_rules) == 1: + raise last_exception + else: + raise ParseError( + last_error_pos, + 'Expected %s but got %s', + ','.join(last_error_rules), + self.text[last_error_pos] + ) + + def maybe_char(self, chars=None): + try: + return self.char(chars) + except ParseError: + return None + + def maybe_match(self, *rules): + try: + return self.match(*rules) + except ParseError: + return None + + def maybe_keyword(self, *keywords): + try: + return self.keyword(*keywords) + except ParseError: + return None + +class CalcParser(Parser): + def start(self): + return self.expression() + + def expression(self): + rv = self.match('term') + while True: + op = self.maybe_keyword('+', '-') + if op is None: + break + + term = self.match('term') + if op == '+': + rv += term + else: + rv -= term + + return rv + + def term(self): + rv = self.match('factor') + while True: + op = self.maybe_keyword('*', '/') + if op is None: + break + + term = self.match('factor') + if op == '*': + rv *= term + else: + rv /= term + + return rv + + def factor(self): + if self.maybe_keyword('('): + rv = self.match('expression') + self.keyword(')') + + return rv + + return self.match('number') + + def number(self): + chars = [] + + sign = self.maybe_keyword('+', '-') + if sign is not None: + chars.append(sign) + + chars.append(self.char('0-9')) + + while True: + char = self.maybe_char('0-9') + if char is None: + break + + chars.append(char) + + if self.maybe_char('.'): + chars.append('.') + chars.append(self.char('0-9')) + + while True: + char = self.maybe_char('0-9') + if char is None: + break + + chars.append(char) + + rv = float(''.join(chars)) + return rv + diff --git a/ebsynth_utility/ebsynth_utility.py b/ebsynth_utility/ebsynth_utility.py new file mode 100644 index 0000000000000000000000000000000000000000..68c7466a81389c88f968086b58f14a6eb05f4870 --- /dev/null +++ b/ebsynth_utility/ebsynth_utility.py @@ -0,0 +1,185 @@ +import os + +from modules.ui import plaintext_to_html + +import cv2 +import glob +from PIL import Image + +from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert +from extensions.ebsynth_utility.stage2 import ebsynth_utility_stage2 +from extensions.ebsynth_utility.stage5 import ebsynth_utility_stage5 +from extensions.ebsynth_utility.stage7 import ebsynth_utility_stage7 +from extensions.ebsynth_utility.stage8 import ebsynth_utility_stage8 +from extensions.ebsynth_utility.stage3_5 import ebsynth_utility_stage3_5 + + +def x_ceiling(value, step): + return -(-value // step) * step + +def dump_dict(string, d:dict): + for key in d.keys(): + string += ( key + " : " + str(d[key]) + "\n") + return string + +class debug_string: + txt = "" + def print(self, comment): + print(comment) + self.txt += comment + '\n' + def to_string(self): + return self.txt + +def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, frame_width:int, frame_height:int, st1_masking_method_index:int, st1_mask_threshold:float, tb_use_fast_mode:bool, tb_use_jit:bool, clipseg_mask_prompt:str, clipseg_exclude_prompt:str, clipseg_mask_threshold:int, clipseg_mask_blur_size:int, clipseg_mask_blur_size2:int, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, color_matcher_method:str, st3_5_use_mask:bool, st3_5_use_mask_ref:bool, st3_5_use_mask_org:bool, color_matcher_ref_type:int, color_matcher_ref_image:Image, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_threshold:float, fg_transparency:float, mask_mode:str): + args = locals() + info = "" + info = dump_dict(info, args) + dbg = debug_string() + + + def process_end(dbg, info): + return plaintext_to_html(dbg.to_string()), plaintext_to_html(info) + + + if not os.path.isdir(project_dir): + dbg.print("{0} project_dir not found".format(project_dir)) + return process_end( dbg, info ) + + if not os.path.isfile(original_movie_path): + dbg.print("{0} original_movie_path not found".format(original_movie_path)) + return process_end( dbg, info ) + + is_invert_mask = False + if mask_mode == "Invert": + is_invert_mask = True + + frame_path = os.path.join(project_dir , "video_frame") + frame_mask_path = os.path.join(project_dir, "video_mask") + + if is_invert_mask: + inv_path = os.path.join(project_dir, "inv") + os.makedirs(inv_path, exist_ok=True) + + org_key_path = os.path.join(inv_path, "video_key") + img2img_key_path = os.path.join(inv_path, "img2img_key") + img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key") + else: + org_key_path = os.path.join(project_dir, "video_key") + img2img_key_path = os.path.join(project_dir, "img2img_key") + img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key") + + if mask_mode == "None": + frame_mask_path = "" + + + project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path] + + + if stage_index == 0: + ebsynth_utility_stage1(dbg, project_args, frame_width, frame_height, st1_masking_method_index, st1_mask_threshold, tb_use_fast_mode, tb_use_jit, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2, is_invert_mask) + if is_invert_mask: + inv_mask_path = os.path.join(inv_path, "inv_video_mask") + ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path) + + elif stage_index == 1: + ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask) + elif stage_index == 2: + + sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] + img_height, img_width, _ = cv2.imread(sample_image).shape + if img_width < img_height: + re_w = 512 + re_h = int(x_ceiling( (512 / img_width) * img_height , 64)) + else: + re_w = int(x_ceiling( (512 / img_height) * img_width , 64)) + re_h = 512 + img_width = re_w + img_height = re_h + + dbg.print("stage 3") + dbg.print("") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("1. Go to img2img tab") + dbg.print("2. Select [ebsynth utility] in the script combo box") + dbg.print("3. Fill in the \"Project directory\" field with [" + project_dir + "]" ) + dbg.print("4. Select in the \"Mask Mode(Override img2img Mask mode)\" field with [" + ("Invert" if is_invert_mask else "Normal") + "]" ) + dbg.print("5. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" ) + dbg.print("6. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" ) + dbg.print("7. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" ) + dbg.print(" (When using controlnet together, you can put in large values (even 1.0 is possible).)") + dbg.print("8. Fill in the remaining configuration fields of img2img. No image and mask settings are required.") + dbg.print("9. Drop any image onto the img2img main screen. This is necessary to avoid errors, but does not affect the results of img2img.") + dbg.print("10. Generate") + dbg.print("(Images are output to [" + img2img_key_path + "])") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + + elif stage_index == 3: + ebsynth_utility_stage3_5(dbg, project_args, color_matcher_method, st3_5_use_mask, st3_5_use_mask_ref, st3_5_use_mask_org, color_matcher_ref_type, color_matcher_ref_image) + + elif stage_index == 4: + sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0] + img_height, img_width, _ = cv2.imread(sample_image).shape + + sample_img2img_key = glob.glob( os.path.join(img2img_key_path , "*.png" ) )[0] + img_height_key, img_width_key, _ = cv2.imread(sample_img2img_key).shape + + if is_invert_mask: + project_dir = inv_path + + dbg.print("stage 4") + dbg.print("") + + if img_height == img_height_key and img_width == img_width_key: + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("!! The size of frame and img2img_key matched.") + dbg.print("!! You can skip this stage.") + + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("0. Enable the following item") + dbg.print("Settings ->") + dbg.print(" Saving images/grids ->") + dbg.print(" Use original name for output filename during batch process in extras tab") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir)) + dbg.print("2. Go to Extras tab") + dbg.print("3. Go to Batch from Directory tab") + dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" ) + dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" ) + dbg.print("6. Go to Scale to tab") + dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" ) + dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" ) + dbg.print("9. Fill in the remaining configuration fields of Upscaler.") + dbg.print("10. Generate") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + elif stage_index == 5: + ebsynth_utility_stage5(dbg, project_args, is_invert_mask) + elif stage_index == 6: + + if is_invert_mask: + project_dir = inv_path + + dbg.print("stage 6") + dbg.print("") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("Running ebsynth.(on your self)") + dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir)) + dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir)) + dbg.print("If multiple .ebs files are generated, run them all.") + dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + elif stage_index == 7: + ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask) + elif stage_index == 8: + if mask_mode != "Normal": + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.") + dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + return process_end( dbg, "" ) + ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type) + else: + pass + + return process_end( dbg, info ) diff --git a/ebsynth_utility/imgs/clipseg.png b/ebsynth_utility/imgs/clipseg.png new file mode 100644 index 0000000000000000000000000000000000000000..4e68833759a200695ef09d6e8d08ec724f2b586b Binary files /dev/null and b/ebsynth_utility/imgs/clipseg.png differ diff --git a/ebsynth_utility/imgs/controlnet_0.png b/ebsynth_utility/imgs/controlnet_0.png new file mode 100644 index 0000000000000000000000000000000000000000..7dc7b73e6f9576cee696c1878901048d4d6ff929 Binary files /dev/null and b/ebsynth_utility/imgs/controlnet_0.png differ diff --git a/ebsynth_utility/imgs/controlnet_1.png b/ebsynth_utility/imgs/controlnet_1.png new file mode 100644 index 0000000000000000000000000000000000000000..7560b4f4310424376b5c732608002ee1161bf3fc Binary files /dev/null and b/ebsynth_utility/imgs/controlnet_1.png differ diff --git a/ebsynth_utility/imgs/controlnet_option_in_ebsynthutil.png b/ebsynth_utility/imgs/controlnet_option_in_ebsynthutil.png new file mode 100644 index 0000000000000000000000000000000000000000..0094b752beb0e568321555eafd9db0f948b641fc Binary files /dev/null and b/ebsynth_utility/imgs/controlnet_option_in_ebsynthutil.png differ diff --git a/ebsynth_utility/imgs/controlnet_setting.png b/ebsynth_utility/imgs/controlnet_setting.png new file mode 100644 index 0000000000000000000000000000000000000000..cc53d786afa0d7a09cdd2d87fd0e036cde69dd2e Binary files /dev/null and b/ebsynth_utility/imgs/controlnet_setting.png differ diff --git a/ebsynth_utility/imgs/sample1.mp4 b/ebsynth_utility/imgs/sample1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..00cad979c7a11aed65e5959aec18ec60f167156c --- /dev/null +++ b/ebsynth_utility/imgs/sample1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5458eea82a691a584af26f51d1db728092069a3409c6f9eb2dd14fd2b71173 +size 4824162 diff --git a/ebsynth_utility/imgs/sample2.mp4 b/ebsynth_utility/imgs/sample2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e429339273350c132c563a1da76d212f9212b4ed --- /dev/null +++ b/ebsynth_utility/imgs/sample2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537b8331b74d8ea49ee580aed138d460735ef897ab31ca694031d7a56d99ff72 +size 2920523 diff --git a/ebsynth_utility/imgs/sample3.mp4 b/ebsynth_utility/imgs/sample3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e64e52d5ff97864046e7d3bfe975d74738aafd99 --- /dev/null +++ b/ebsynth_utility/imgs/sample3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49739c2ef1f2ecaf14453f463e46b1a05de1688ce22b50200563a03b1758ddf +size 5161880 diff --git a/ebsynth_utility/imgs/sample4.mp4 b/ebsynth_utility/imgs/sample4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb5965b9573d430d27cbc9bcaf1c73c838a1d45c --- /dev/null +++ b/ebsynth_utility/imgs/sample4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62ed25263b6e5328a49460714c0b6e6ac46759921c87c91850f749f5bf068cfa +size 5617838 diff --git a/ebsynth_utility/imgs/sample5.mp4 b/ebsynth_utility/imgs/sample5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..670b7c583cdbb35d340282ed3b7cb59d87c91649 --- /dev/null +++ b/ebsynth_utility/imgs/sample5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e429cfef8b3ed7829ce3219895fcfdbbe94e1494a2e4dcd87988e03509c8d50 +size 4190467 diff --git a/ebsynth_utility/imgs/sample6.mp4 b/ebsynth_utility/imgs/sample6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f485284f2d21bd38c31f6d23b991f856239f08b --- /dev/null +++ b/ebsynth_utility/imgs/sample6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9a9ea8662ef1b7fc7151b987006eef8dd3598e320242bd87a2838ac8733df6 +size 6890883 diff --git a/ebsynth_utility/imgs/sample_anyaheh.mp4 b/ebsynth_utility/imgs/sample_anyaheh.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f8efdac7ecf4fd7888ee3f676dcc4a9afc6d8624 --- /dev/null +++ b/ebsynth_utility/imgs/sample_anyaheh.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4de4e9b9758cefe28c430f909da9dfc086e5b3510e9d0aa7becab7b4be355447 +size 12159686 diff --git a/ebsynth_utility/imgs/sample_autotag.mp4 b/ebsynth_utility/imgs/sample_autotag.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b461523d5228a8dd1e2ce8c1957578730a3c6045 --- /dev/null +++ b/ebsynth_utility/imgs/sample_autotag.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e904d68fc0fd9ac09ce153a9d54e9f1ce9f8db7cf5e96109c496f7e64924c92 +size 7058129 diff --git a/ebsynth_utility/imgs/sample_clipseg.mp4 b/ebsynth_utility/imgs/sample_clipseg.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c009c2127273ed15ca701f6bb119ba053d88dc0 --- /dev/null +++ b/ebsynth_utility/imgs/sample_clipseg.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17701c5c3a376d3c4cf8ce0acfb991033830d56670ca3178eedd6e671e096af3 +size 10249706 diff --git a/ebsynth_utility/install.py b/ebsynth_utility/install.py new file mode 100644 index 0000000000000000000000000000000000000000..018d5000add4858cee9d4a9a9bf95112a60238f9 --- /dev/null +++ b/ebsynth_utility/install.py @@ -0,0 +1,24 @@ +import launch + +def update_transparent_background(): + from importlib.metadata import version as meta_version + from packaging import version + v = meta_version("transparent-background") + print("current transparent-background " + v) + if version.parse(v) < version.parse('1.2.3'): + launch.run_pip("install -U transparent-background", "update transparent-background version for Ebsynth Utility") + +if not launch.is_installed("transparent_background"): + launch.run_pip("install transparent-background", "requirements for Ebsynth Utility") + +update_transparent_background() + +if not launch.is_installed("IPython"): + launch.run_pip("install ipython", "requirements for Ebsynth Utility") + +if not launch.is_installed("seaborn"): + launch.run_pip("install ""seaborn>=0.11.0""", "requirements for Ebsynth Utility") + +if not launch.is_installed("color_matcher"): + launch.run_pip("install color-matcher", "requirements for Ebsynth Utility") + diff --git a/ebsynth_utility/sample/add_token.txt b/ebsynth_utility/sample/add_token.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f413582f6f4888c4baf01324e1bc71ad734c6bf --- /dev/null +++ b/ebsynth_utility/sample/add_token.txt @@ -0,0 +1,54 @@ +[ + { + "target":"smile", + "min_score":0.5, + "token": ["lottalewds_v0", "1.2"], + "type":"lora" + }, + { + "target":"smile", + "min_score":0.5, + "token": ["anyahehface", "score*1.2"], + "type":"normal" + }, + { + "target":"smile", + "min_score":0.5, + "token": ["wicked smug", "score*1.2"], + "type":"normal" + }, + { + "target":"smile", + "min_score":0.5, + "token": ["half closed eyes", "0.2 + score*0.3"], + "type":"normal" + }, + + + + { + "target":"test_token", + "min_score":0.8, + "token": ["lora_name_A", "0.5"], + "type":"lora" + }, + { + "target":"test_token", + "min_score":0.5, + "token": ["bbbb", "score - 0.1"], + "type":"normal" + }, + { + "target":"test_token2", + "min_score":0.8, + "token": ["hypernet_name_A", "score"], + "type":"hypernet" + }, + { + "target":"test_token3", + "min_score":0.0, + "token": ["dddd", "score"], + "type":"normal" + } +] + diff --git a/ebsynth_utility/sample/blacklist.txt b/ebsynth_utility/sample/blacklist.txt new file mode 100644 index 0000000000000000000000000000000000000000..6938ecd2e4eb99ad6403d077a4678e633c710d1e --- /dev/null +++ b/ebsynth_utility/sample/blacklist.txt @@ -0,0 +1,10 @@ +motion_blur +blurry +realistic +depth_of_field +mountain +tree +water +underwater +tongue +tongue_out diff --git a/ebsynth_utility/scripts/__pycache__/custom_script.cpython-310.pyc b/ebsynth_utility/scripts/__pycache__/custom_script.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..650293490ae83b91af39906bfd9bc9b7024acfaf Binary files /dev/null and b/ebsynth_utility/scripts/__pycache__/custom_script.cpython-310.pyc differ diff --git a/ebsynth_utility/scripts/__pycache__/ui.cpython-310.pyc b/ebsynth_utility/scripts/__pycache__/ui.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc2deee3fda2d158a1511131a1992d00f06ad623 Binary files /dev/null and b/ebsynth_utility/scripts/__pycache__/ui.cpython-310.pyc differ diff --git a/ebsynth_utility/scripts/custom_script.py b/ebsynth_utility/scripts/custom_script.py new file mode 100644 index 0000000000000000000000000000000000000000..661d7e5d68f762d28bfc58d55ca3cb306ae392f7 --- /dev/null +++ b/ebsynth_utility/scripts/custom_script.py @@ -0,0 +1,1012 @@ +import modules.scripts as scripts +import gradio as gr +import os +import torch +import random +import time +import pprint +import shutil + +from modules.processing import process_images,Processed +from modules.paths import models_path +from modules.textual_inversion import autocrop +import modules.images +from modules import shared,deepbooru,masking +import cv2 +import copy +import numpy as np +from PIL import Image,ImageOps +import glob +import requests +import json +import re +from extensions.ebsynth_utility.calculator import CalcParser,ParseError + +def get_my_dir(): + if os.path.isdir("extensions/ebsynth_utility"): + return "extensions/ebsynth_utility" + return scripts.basedir() + +def x_ceiling(value, step): + return -(-value // step) * step + +def remove_pngs_in_dir(path): + if not os.path.isdir(path): + return + pngs = glob.glob( os.path.join(path, "*.png") ) + for png in pngs: + os.remove(png) + +def resize_img(img, w, h): + if img.shape[0] + img.shape[1] < h + w: + interpolation = interpolation=cv2.INTER_CUBIC + else: + interpolation = interpolation=cv2.INTER_AREA + + return cv2.resize(img, (w, h), interpolation=interpolation) + +def download_and_cache_models(dirname): + download_url = 'https://github.com/zymk9/yolov5_anime/blob/8b50add22dbd8224904221be3173390f56046794/weights/yolov5s_anime.pt?raw=true' + model_file_name = 'yolov5s_anime.pt' + + if not os.path.exists(dirname): + os.makedirs(dirname) + + cache_file = os.path.join(dirname, model_file_name) + if not os.path.exists(cache_file): + print(f"downloading face detection model from '{download_url}' to '{cache_file}'") + response = requests.get(download_url) + with open(cache_file, "wb") as f: + f.write(response.content) + + if os.path.exists(cache_file): + return cache_file + return None + +class Script(scripts.Script): + anime_face_detector = None + face_detector = None + face_merge_mask_filename = "face_crop_img2img_mask.png" + face_merge_mask_image = None + prompts_dir = "" + calc_parser = None + is_invert_mask = False + controlnet_weight = 0.5 + controlnet_weight_for_face = 0.5 + add_tag_replace_underscore = False + + +# The title of the script. This is what will be displayed in the dropdown menu. + def title(self): + return "ebsynth utility" + +# Determines when the script should be shown in the dropdown menu via the +# returned value. As an example: +# is_img2img is True if the current tab is img2img, and False if it is txt2img. +# Thus, return is_img2img to only show the script on the img2img tab. + + def show(self, is_img2img): + return is_img2img + +# How the script's is displayed in the UI. See https://gradio.app/docs/#components +# for the different UI components you can use and how to create them. +# Most UI components can return a value, such as a boolean for a checkbox. +# The returned values are passed to the run method as parameters. + + def ui(self, is_img2img): + with gr.Column(variant='panel'): + with gr.Column(): + project_dir = gr.Textbox(label='Project directory', lines=1) + generation_test = gr.Checkbox(False, label="Generation TEST!!(Ignore Project directory and use the image and mask specified in the main UI)") + + with gr.Accordion("Mask option"): + mask_mode = gr.Dropdown(choices=["Normal","Invert","None","Don't Override"], value="Normal" ,label="Mask Mode(Override img2img Mask mode)") + inpaint_area = gr.Dropdown(choices=["Whole picture","Only masked","Don't Override"], type = "index", value="Only masked" ,label="Inpaint Area(Override img2img Inpaint area)") + use_depth = gr.Checkbox(True, label="Use Depth Map If exists in /video_key_depth") + gr.HTML(value="
\ + See \ + [here] for depth map.\ +
") + + with gr.Accordion("ControlNet option"): + controlnet_weight = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight") + controlnet_weight_for_face = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight For Face") + use_preprocess_img = gr.Checkbox(True, label="Use Preprocess image If exists in /controlnet_preprocess") + gr.HTML(value="\
+ Please enable the following settings to use controlnet from this script.
\
+ \
+ Settings->ControlNet->Allow other script to control this extension\
+ \
+
\
+ The results are stored in timestamp_prompts.txt.
\
+ If you want to use the same tagging results the next time you run img2img, rename the file to prompts.txt
\
+ Recommend enabling the following settings.
\
+ \
+ Settings->Interrogate Option->Interrogate: include ranks of model tags matches in results\
+ \
+
\ + If loading of the Yolov5_anime model fails, check\ + [this] solution.\ +
") + face_crop_resolution = gr.Slider(minimum=128, maximum=2048, step=1, value=512, label="Face Crop Resolution") + max_crop_size = gr.Slider(minimum=0, maximum=2048, step=1, value=1024, label="Max Crop Size") + face_denoising_strength = gr.Slider(minimum=0.00, maximum=1.00, step=0.01, value=0.5, label="Face Denoising Strength") + face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification ") + disable_facecrop_lpbk_last_time = gr.Checkbox(False, label="Disable at the last loopback time") + + with gr.Column(): + enable_face_prompt = gr.Checkbox(False, label="Enable Face Prompt") + face_prompt = gr.Textbox(label="Face Prompt", show_label=False, lines=2, + placeholder="Prompt for Face", + value = "face close up," + ) + + return [project_dir, generation_test, mask_mode, inpaint_area, use_depth, img2img_repeat_count, inc_seed, auto_tag_mode, add_tag_to_head, add_tag_replace_underscore, is_facecrop, face_detection_method, face_crop_resolution, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_weight, controlnet_weight_for_face, disable_facecrop_lpbk_last_time,use_preprocess_img] + + + def detect_face_from_img(self, img_array): + if not self.face_detector: + dnn_model_path = autocrop.download_and_cache_models(os.path.join(models_path, "opencv")) + self.face_detector = cv2.FaceDetectorYN.create(dnn_model_path, "", (0, 0)) + + self.face_detector.setInputSize((img_array.shape[1], img_array.shape[0])) + _, result = self.face_detector.detect(img_array) + return result + + def detect_anime_face_from_img(self, img_array): + import sys + + if not self.anime_face_detector: + if 'models' in sys.modules: + del sys.modules['models'] + + anime_model_path = download_and_cache_models(os.path.join(models_path, "yolov5_anime")) + + if not os.path.isfile(anime_model_path): + print( "WARNING!! " + anime_model_path + " not found.") + print( "use YuNet instead.") + return self.detect_face_from_img(img_array) + + self.anime_face_detector = torch.hub.load('ultralytics/yolov5', 'custom', path=anime_model_path) + + # warmup + test = np.zeros([512,512,3],dtype=np.uint8) + _ = self.anime_face_detector(test) + + result = self.anime_face_detector(img_array) + #models.common.Detections + faces = [] + for x_c, y_c, w, h, _, _ in result.xywh[0].tolist(): + faces.append( [ x_c - w/2 , y_c - h/2, w, h ] ) + + return faces + + def detect_face(self, img, mask, face_detection_method, max_crop_size): + img_array = np.array(img) + + # image without alpha + if img_array.shape[2] == 4: + img_array = img_array[:,:,:3] + + if mask is not None: + if self.is_invert_mask: + mask = ImageOps.invert(mask) + mask_array = np.array(mask)/255 + if mask_array.ndim == 2: + mask_array = mask_array[:, :, np.newaxis] + + if mask_array.shape[2] == 4: + mask_array = mask_array[:,:,:3] + + img_array = mask_array * img_array + img_array = img_array.astype(np.uint8) + + if face_detection_method == "YuNet": + faces = self.detect_face_from_img(img_array) + elif face_detection_method == "Yolov5_anime": + faces = self.detect_anime_face_from_img(img_array) + else: + faces = self.detect_face_from_img(img_array) + + if faces is None or len(faces) == 0: + return [] + + face_coords = [] + for face in faces: + x = int(face[0]) + y = int(face[1]) + w = int(face[2]) + h = int(face[3]) + if max(w,h) > max_crop_size: + print("ignore big face") + continue + if w == 0 or h == 0: + print("ignore w,h = 0 face") + continue + + face_coords.append( [ x/img_array.shape[1],y/img_array.shape[0],w/img_array.shape[1],h/img_array.shape[0]] ) + + return face_coords + + def get_mask(self): + def create_mask( output, x_rate, y_rate, k_size ): + img = np.zeros((512, 512, 3)) + img = cv2.ellipse(img, ((256, 256), (int(512 * x_rate), int(512 * y_rate)), 0), (255, 255, 255), thickness=-1) + img = cv2.GaussianBlur(img, (k_size, k_size), 0) + cv2.imwrite(output, img) + + if self.face_merge_mask_image is None: + mask_file_path = os.path.join( get_my_dir() , self.face_merge_mask_filename) + if not os.path.isfile(mask_file_path): + create_mask( mask_file_path, 0.9, 0.9, 91) + + m = cv2.imread( mask_file_path )[:,:,0] + m = m[:, :, np.newaxis] + self.face_merge_mask_image = m / 255 + + return self.face_merge_mask_image + + def face_img_crop(self, img, face_coords,face_area_magnification): + img_array = np.array(img) + face_imgs =[] + new_coords = [] + + for face in face_coords: + x = int(face[0] * img_array.shape[1]) + y = int(face[1] * img_array.shape[0]) + w = int(face[2] * img_array.shape[1]) + h = int(face[3] * img_array.shape[0]) + print([x,y,w,h]) + + cx = x + int(w/2) + cy = y + int(h/2) + + x = cx - int(w*face_area_magnification / 2) + x = x if x > 0 else 0 + w = cx + int(w*face_area_magnification / 2) - x + w = w if x+w < img.width else img.width - x + + y = cy - int(h*face_area_magnification / 2) + y = y if y > 0 else 0 + h = cy + int(h*face_area_magnification / 2) - y + h = h if y+h < img.height else img.height - y + + print([x,y,w,h]) + + face_imgs.append( img_array[y: y+h, x: x+w] ) + new_coords.append( [x,y,w,h] ) + + resized = [] + for face_img in face_imgs: + if face_img.shape[1] < face_img.shape[0]: + re_w = self.face_crop_resolution + re_h = int(x_ceiling( (self.face_crop_resolution / face_img.shape[1]) * face_img.shape[0] , 64)) + else: + re_w = int(x_ceiling( (self.face_crop_resolution / face_img.shape[0]) * face_img.shape[1] , 64)) + re_h = self.face_crop_resolution + + face_img = resize_img(face_img, re_w, re_h) + resized.append( Image.fromarray(face_img)) + + return resized, new_coords + + def face_crop_img2img(self, p, face_coords, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_input_img, controlnet_input_face_imgs, preprocess_img_exist): + + def merge_face(img, face_img, face_coord, base_img_size, mask): + x_rate = img.width / base_img_size[0] + y_rate = img.height / base_img_size[1] + + img_array = np.array(img) + x = int(face_coord[0] * x_rate) + y = int(face_coord[1] * y_rate) + w = int(face_coord[2] * x_rate) + h = int(face_coord[3] * y_rate) + + face_array = np.array(face_img) + face_array = resize_img(face_array, w, h) + mask = resize_img(mask, w, h) + if mask.ndim == 2: + mask = mask[:, :, np.newaxis] + + bg = img_array[y: y+h, x: x+w] + img_array[y: y+h, x: x+w] = mask * face_array + (1-mask)*bg + + return Image.fromarray(img_array) + + base_img = p.init_images[0] + + base_img_size = (base_img.width, base_img.height) + + if face_coords is None or len(face_coords) == 0: + print("no face detected") + return process_images(p) + + print(face_coords) + face_imgs, new_coords = self.face_img_crop(base_img, face_coords, face_area_magnification) + + if not face_imgs: + return process_images(p) + + face_p = copy.copy(p) + + ### img2img base img + proc = self.process_images(p, controlnet_input_img, self.controlnet_weight, preprocess_img_exist) + print(proc.seed) + + ### img2img for each face + face_img2img_results = [] + + for face, coord, controlnet_input_face in zip(face_imgs, new_coords, controlnet_input_face_imgs): + # cv2.imwrite("scripts/face.png", np.array(face)[:, :, ::-1]) + face_p.init_images = [face] + face_p.width = face.width + face_p.height = face.height + face_p.denoising_strength = face_denoising_strength + + if enable_face_prompt: + face_p.prompt = face_prompt + else: + face_p.prompt = "close-up face ," + face_p.prompt + + if p.image_mask is not None: + x,y,w,h = coord + cropped_face_mask = Image.fromarray(np.array(p.image_mask)[y: y+h, x: x+w]) + face_p.image_mask = modules.images.resize_image(0, cropped_face_mask, face.width, face.height) + + face_proc = self.process_images(face_p, controlnet_input_face, self.controlnet_weight_for_face, preprocess_img_exist) + print(face_proc.seed) + + face_img2img_results.append((face_proc.images[0], coord)) + + ### merge faces + bg = proc.images[0] + mask = self.get_mask() + + for face_img, coord in face_img2img_results: + bg = merge_face(bg, face_img, coord, base_img_size, mask) + + proc.images[0] = bg + + return proc + + def get_depth_map(self, mask, depth_path ,img_basename, is_invert_mask): + depth_img_path = os.path.join( depth_path , img_basename ) + + depth = None + + if os.path.isfile( depth_img_path ): + depth = Image.open(depth_img_path) + else: + # try 00001-0000.png + os.path.splitext(img_basename)[0] + depth_img_path = os.path.join( depth_path , os.path.splitext(img_basename)[0] + "-0000.png" ) + if os.path.isfile( depth_img_path ): + depth = Image.open(depth_img_path) + + if depth: + if mask: + mask_array = np.array(mask) + depth_array = np.array(depth) + + if is_invert_mask == False: + depth_array[mask_array[:,:,0] == 0] = 0 + else: + depth_array[mask_array[:,:,0] != 0] = 0 + + depth = Image.fromarray(depth_array) + + tmp_path = os.path.join( depth_path , "tmp" ) + os.makedirs(tmp_path, exist_ok=True) + tmp_path = os.path.join( tmp_path , img_basename ) + depth_array = depth_array.astype(np.uint16) + cv2.imwrite(tmp_path, depth_array) + + mask = depth + + return depth!=None, mask + +### auto tagging + debug_count = 0 + + def get_masked_image(self, image, mask_image): + + if mask_image == None: + return image.convert("RGB") + + mask = mask_image.convert('L') + if self.is_invert_mask: + mask = ImageOps.invert(mask) + crop_region = masking.get_crop_region(np.array(mask), 0) +# crop_region = masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height) +# x1, y1, x2, y2 = crop_region + image = image.crop(crop_region).convert("RGB") + mask = mask.crop(crop_region) + + base_img = Image.new("RGB", image.size, (255, 190, 200)) + + image = Image.composite( image, base_img, mask ) + +# image.save("scripts/get_masked_image_test_"+ str(self.debug_count) + ".png") +# self.debug_count += 1 + + return image + + def interrogate_deepdanbooru(self, imgs, masks): + prompts_dict = {} + cause_err = False + + try: + deepbooru.model.start() + + for img,mask in zip(imgs,masks): + key = os.path.basename(img) + print(key + " interrogate deepdanbooru") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + prompt = deepbooru.model.tag_multi(image) + + prompts_dict[key] = prompt + except Exception as e: + import traceback + traceback.print_exc() + print(e) + cause_err = True + finally: + deepbooru.model.stop() + if cause_err: + print("Exception occurred during auto-tagging(deepdanbooru)") + return Processed() + + return prompts_dict + + + def interrogate_clip(self, imgs, masks): + from modules import devices, shared, lowvram, paths + import importlib + import models + + caption_list = [] + prompts_dict = {} + cause_err = False + + try: + if shared.cmd_opts.lowvram or shared.cmd_opts.medvram: + lowvram.send_everything_to_cpu() + devices.torch_gc() + + with paths.Prioritize("BLIP"): + importlib.reload(models) + shared.interrogator.load() + + for img,mask in zip(imgs,masks): + key = os.path.basename(img) + print(key + " generate caption") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + caption = shared.interrogator.generate_caption(image) + caption_list.append(caption) + + shared.interrogator.send_blip_to_ram() + devices.torch_gc() + + for img,mask,caption in zip(imgs,masks,caption_list): + key = os.path.basename(img) + print(key + " interrogate clip") + + image = Image.open(img) + mask_image = Image.open(mask) if mask else None + image = self.get_masked_image(image, mask_image) + + clip_image = shared.interrogator.clip_preprocess(image).unsqueeze(0).type(shared.interrogator.dtype).to(devices.device_interrogate) + + res = "" + + with torch.no_grad(), devices.autocast(): + image_features = shared.interrogator.clip_model.encode_image(clip_image).type(shared.interrogator.dtype) + image_features /= image_features.norm(dim=-1, keepdim=True) + + for name, topn, items in shared.interrogator.categories(): + matches = shared.interrogator.rank(image_features, items, top_count=topn) + for match, score in matches: + if shared.opts.interrogate_return_ranks: + res += f", ({match}:{score/100:.3f})" + else: + res += ", " + match + + prompts_dict[key] = (caption + res) + + except Exception as e: + import traceback + traceback.print_exc() + print(e) + cause_err = True + finally: + shared.interrogator.unload() + if cause_err: + print("Exception occurred during auto-tagging(blip/clip)") + return Processed() + + return prompts_dict + + + def remove_reserved_token(self, token_list): + reserved_list = ["pink_background","simple_background","pink","pink_theme"] + + result_list = [] + + head_token = token_list[0] + + if head_token[2] == "normal": + head_token_str = head_token[0].replace('pink background', '') + token_list[0] = (head_token_str, head_token[1], head_token[2]) + + for token in token_list: + if token[0] in reserved_list: + continue + result_list.append(token) + + return result_list + + def remove_blacklisted_token(self, token_list): + black_list_path = os.path.join(self.prompts_dir, "blacklist.txt") + if not os.path.isfile(black_list_path): + print(black_list_path + " not found.") + return token_list + + with open(black_list_path) as f: + black_list = [s.strip() for s in f.readlines()] + + result_list = [] + + for token in token_list: + if token[0] in black_list: + continue + result_list.append(token) + + token_list = result_list + + return token_list + + def add_token(self, token_list): + add_list_path = os.path.join(self.prompts_dir, "add_token.txt") + if not os.path.isfile(add_list_path): + print(add_list_path + " not found.") + + if self.add_tag_replace_underscore: + token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ] + + return token_list + + if not self.calc_parser: + self.calc_parser = CalcParser() + + with open(add_list_path) as f: + add_list = json.load(f) + ''' + [ + { + "target":"test_token", + "min_score":0.8, + "token": ["lora_name_A", "0.5"], + "type":"lora" + }, + { + "target":"test_token", + "min_score":0.5, + "token": ["bbbb", "score - 0.1"], + "type":"normal" + }, + { + "target":"test_token2", + "min_score":0.8, + "token": ["hypernet_name_A", "score"], + "type":"hypernet" + }, + { + "target":"test_token3", + "min_score":0.0, + "token": ["dddd", "score"], + "type":"normal" + } + ] + ''' + result_list = [] + + for token in token_list: + for add_item in add_list: + if token[0] == add_item["target"]: + if token[1] > add_item["min_score"]: + # hit + formula = str(add_item["token"][1]) + formula = formula.replace("score",str(token[1])) + print('Input: %s' % str(add_item["token"][1])) + + try: + score = self.calc_parser.parse(formula) + score = round(score, 3) + except (ParseError, ZeroDivisionError) as e: + print('Input: %s' % str(add_item["token"][1])) + print('Error: %s' % e) + print("ignore this token") + continue + + print("score = " + str(score)) + result_list.append( ( add_item["token"][0], score, add_item["type"] ) ) + + if self.add_tag_replace_underscore: + token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ] + + token_list = token_list + result_list + + return token_list + + def create_prompts_dict(self, imgs, masks, auto_tag_mode): + prompts_dict = {} + + if auto_tag_mode == "DeepDanbooru": + raw_dict = self.interrogate_deepdanbooru(imgs, masks) + elif auto_tag_mode == "CLIP": + raw_dict = self.interrogate_clip(imgs, masks) + + repatter = re.compile(r'\((.+)\:([0-9\.]+)\)') + + for key, value_str in raw_dict.items(): + value_list = [x.strip() for x in value_str.split(',')] + + value = [] + for v in value_list: + m = repatter.fullmatch(v) + if m: + value.append((m.group(1), float(m.group(2)), "normal")) + else: + value.append((v, 1, "no_score")) + +# print(value) + value = self.remove_reserved_token(value) +# print(value) + value = self.remove_blacklisted_token(value) +# print(value) + value = self.add_token(value) +# print(value) + + def create_token_str(x): + print(x) + if x[2] == "no_score": + return x[0] + elif x[2] == "lora": + return "\ + If you have trouble entering the video path manually, you can also use drag and drop.For large videos, please enter the path manually. \ +
") + + with gr.TabItem('configuration', elem_id='ebs_configuration'): + with gr.Tabs(elem_id="ebs_configuration_tab"): + with gr.TabItem(label="stage 1",elem_id='ebs_configuration_tab1'): + with gr.Row(): + frame_width = gr.Number(value=-1, label="Frame Width", precision=0, interactive=True) + frame_height = gr.Number(value=-1, label="Frame Height", precision=0, interactive=True) + gr.HTML(value="\ + -1 means that it is calculated automatically. If both are -1, the size will be the same as the source size. \ +
") + + st1_masking_method_index = gr.Radio(label='Masking Method', choices=["transparent-background","clipseg","transparent-background AND clipseg"], value="transparent-background", type="index") + + with gr.Accordion(label="transparent-background options"): + st1_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) + + # https://pypi.org/project/transparent-background/ + gr.HTML(value="\ + configuration for \ + [transparent-background]\ +
") + tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False) + tb_use_jit = gr.Checkbox(label="Use Jit", value=False) + + with gr.Accordion(label="clipseg options"): + clipseg_mask_prompt = gr.Textbox(label='Mask Target (e.g., girl, cats)', lines=1) + clipseg_exclude_prompt = gr.Textbox(label='Exclude Target (e.g., finger, book)', lines=1) + clipseg_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.4) + clipseg_mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(MedianBlur)', value=11) + clipseg_mask_blur_size2 = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(GaussianBlur)', value=11) + + with gr.TabItem(label="stage 2", elem_id='ebs_configuration_tab2'): + key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10) + key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300) + key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5) + key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True) + + with gr.TabItem(label="stage 3.5", elem_id='ebs_configuration_tab3_5'): + gr.HTML(value="\ + [color-matcher]\ +
") + + color_matcher_method = gr.Radio(label='Color Transfer Method', choices=['default', 'hm', 'reinhard', 'mvgd', 'mkl', 'hm-mvgd-hm', 'hm-mkl-hm'], value="hm-mkl-hm", type="value") + color_matcher_ref_type = gr.Radio(label='Color Matcher Ref Image Type', choices=['original video frame', 'first frame of img2img result'], value="original video frame", type="index") + gr.HTML(value="\ + If an image is specified below, it will be used with highest priority.\ +
") + color_matcher_ref_image = gr.Image(label="Color Matcher Ref Image", source='upload', mirror_webcam=False, type='pil') + st3_5_use_mask = gr.Checkbox(label="Apply mask to the result", value=True) + st3_5_use_mask_ref = gr.Checkbox(label="Apply mask to the Ref Image", value=False) + st3_5_use_mask_org = gr.Checkbox(label="Apply mask to original image", value=False) + #st3_5_number_of_itr = gr.Slider(minimum=1, maximum=10, step=1, label='Number of iterations', value=1) + + with gr.TabItem(label="stage 7", elem_id='ebs_configuration_tab7'): + blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0) + export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type") + + with gr.TabItem(label="stage 8", elem_id='ebs_configuration_tab8'): + bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1) + bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type") + mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5) + mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0) + #is_transparent = gr.Checkbox(label="Is Transparent", value=True, visible = False) + fg_transparency = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Foreground Transparency', value=0.0) + + with gr.TabItem(label="etc", elem_id='ebs_configuration_tab_etc'): + mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode") + + with gr.Column(variant='panel'): + with gr.Column(scale=1): + with gr.Group(): + debug_info = gr.HTML(elem_id="ebs_info_area", value=".") + + with gr.Column(scale=2): + stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 3.5","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index") + gr.HTML(value="\
+ The process of creating a video can be divided into the following stages.
\
+ (Stage 3, 4, and 6 only show a guide and do nothing actual processing.)
\
+ stage 1
\
+ Extract frames from the original video.
\
+ Generate a mask image.
\
+ stage 2
\
+ Select keyframes to be given to ebsynth.
\
+ stage 3
\
+ img2img keyframes.
\
+ stage 3.5
\
+ (this is optional. Perform color correction on the img2img results and expect flickering to decrease. Or, you can simply change the color tone from the generated result.)
\
+ stage 4
\
+ and upscale to the size of the original video.
\
+ stage 5
\
+ Rename keyframes.
\
+ Generate .ebs file.(ebsynth project file)
\
+ stage 6
\
+ Running ebsynth.(on your self)
\
+ Open the generated .ebs under project directory and press [Run All] button.
\
+ If ""out-*"" directory already exists in the Project directory, delete it manually before executing.
\
+ If multiple .ebs files are generated, run them all.
\
+ stage 7
\
+ Concatenate each frame while crossfading.
\
+ Composite audio files extracted from the original video onto the concatenated video.
\
+ stage 8
\
+ This is an extra stage.
\
+ You can put any image or images or video you like in the background.
\
+ You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]
\
+ If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),
\
+ You can specify \"path_to_project_dir/inv/crossfade_tmp\".
\
+