remove dir

sagemaker/fixing/extensions/supermerger/scripts/mergers/mergers.py

@@ -1,1448 +0,0 @@
-import random
-import os
-import gc
-import hashlib
-import numpy as np
-import os.path
-import re
-import torch
-import tqdm
-import datetime
-
-import csv
-import json
-import launch
-import torch.nn as nn
-import scipy.ndimage
-from copy import deepcopy
-from PIL import Image, ImageFont, ImageDraw
-from tqdm import tqdm
-from functools import partial
-from torch import Tensor, lerp
-from torch.nn.functional import cosine_similarity, relu, softplus
-from modules import shared, processing, sd_models, sd_vae, images, sd_samplers, scripts,devices, extras
-from modules.ui import  plaintext_to_html
-from modules.shared import opts
-from modules.processing import create_infotext,Processed
-from modules.sd_models import  load_model,unload_model_weights
-from modules.generation_parameters_copypaste import create_override_settings_dict
-from scripts.mergers.model_util import filenamecutter,savemodel
-from math import ceil
-import sys
-from multiprocessing import cpu_count
-from threading import Lock
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from scripts.mergers.bcolors import bcolors
-import collections
-
-try:
-    ui_version = int(launch.git_tag().split("-",1)[0].replace("v","").replace(".",""))
-except:
-    ui_version = 100
-
-orig_cache = 0
-
-modelcache = collections.OrderedDict()
-
-from inspect import currentframe
-
-SELFKEYS = ["to_out","proj_out","norm"]
-
-module_path = os.path.dirname(os.path.abspath(sys.modules[__name__].__file__))
-scriptpath = os.path.dirname(module_path)
-
-def tryit(func):
-    try:
-        func() 
-    except:
-        pass 
-
-stopmerge = False
-
-def freezemtime():
-    global stopmerge
-    stopmerge = True
-
-mergedmodel=[]
-FINETUNEX = ["IN","OUT","OUT2","CONT","BRI","COL1","COL2","COL3"]
-TYPESEG = ["none","alpha","beta (if Triple or Twice is not selected,Twice automatically enable)","alpha and beta","seed",
-                    "mbw alpha","mbw beta","mbw alpha and beta", "model_A","model_B","model_C","pinpoint blocks (alpha or beta must be selected for another axis)",
-                    "include blocks", "exclude blocks","add include", "add exclude","elemental","add elemental","pinpoint element","effective elemental checker","adjust","pinpoint adjust (IN,OUT,OUT2,CONT,BRI,COL1,COL2,COL3)",
-                    "calcmode","prompt","random"]
-TYPES = ["none","alpha","beta","alpha and beta","seed", "mbw alpha ","mbw beta","mbw alpha and beta",
-                "model_A","model_B","model_C","pinpoint blocks","include blocks","exclude blocks","add include", "add exclude","elemental","add elemental","pinpoint element",
-                "effective","adjust","pinpoint adjust","calcmode","prompt","random"]
-MODES=["Weight" ,"Add" ,"Triple","Twice"]
-SAVEMODES=["save model", "overwrite"]
-EXCLUDE_CHOICES = ["BASE","IN00","IN01","IN02","IN03","IN04","IN05","IN06","IN07","IN08","IN09","IN10","IN11",
-                                  "M00","OUT00","OUT01","OUT02","OUT03","OUT04","OUT05","OUT06","OUT07","OUT08","OUT09","OUT10","OUT11",
-                                  "Adjust","VAE"]           
-CHCKPOINT_DICT_SKIP_ON_MERGE = ["cond_stage_model.transformer.text_model.embeddings.position_ids"]
-
-#type[0:aplha,1:beta,2:seed,3:mbw,4:model_A,5:model_B,6:model_C]
-#msettings=[0 weights_a,1 weights_b,2 model_a,3 model_b,4 model_c,5 base_alpha,6 base_beta,7 mode,8 useblocks,9 custom_name,10 save_sets,11 id_sets,12 wpresets]
-#id sets "image", "PNG info","XY grid"
-
-hear = False
-hearm = False
-NON4 = [None]*4
-
-informer = sd_models.get_closet_checkpoint_match
-
-#msettings=[weights_a,weights_b,model_a,model_b,model_c,device,base_alpha,base_beta,mode,loranames,useblocks,custom_name,save_sets,id_sets,wpresets,deep]  
-
-def smergegen(weights_a,weights_b,model_a,model_b,model_c,base_alpha,base_beta,mode,
-                       calcmode,useblocks,custom_name,save_sets,id_sets,wpresets,deep,tensor,bake_in_vae,opt_value,inex,ex_blocks,ex_elems,
-                       esettings,
-                       s_prompt,s_nprompt,s_steps,s_sampler,s_cfg,s_seed,s_w,s_h,s_batch_size,
-                       genoptions,s_hrupscaler,s_hr2ndsteps,s_denois_str,s_hr_scale,
-                       lmode,lsets,llimits_u,llimits_l,lseed,lserial,lcustom,lround,
-                       currentmodel,imggen,
-                       *txt2imgparams):
-
-    lucks = {"on":False, "mode":lmode,"set":lsets,"upp":llimits_u,"low":llimits_l,"seed":lseed,"num":lserial,"cust":lcustom,"round":int(lround)}
-    deepprint  = "print change" in esettings
-
-    cachedealer(True)
-
-    result,currentmodel,modelid,theta_0,metadata = smerge(
-                        weights_a,weights_b,model_a,model_b,model_c,base_alpha,base_beta,mode,calcmode,
-                        useblocks,custom_name,save_sets,id_sets,wpresets,deep,tensor,bake_in_vae,opt_value,inex,ex_blocks,ex_elems,deepprint,lucks
-                        )
-
-    if "ERROR" in result or "STOPPED" in result: 
-        return result,"not loaded",*NON4
-
-    checkpoint_info = sd_models.get_closet_checkpoint_match(model_a)
-
-    if ui_version >= 150: checkpoint_info = fake_checkpoint_info(checkpoint_info,metadata,currentmodel)
-
-    save = True if SAVEMODES[0] in save_sets else False
-
-    result = savemodel(theta_0,currentmodel,custom_name,save_sets,metadata) if save else "Merged model loaded:"+currentmodel
-
-    sd_models.model_data.__init__()
-    load_model(checkpoint_info, already_loaded_state_dict=theta_0)
-
-    cachedealer(False)
-
-    del theta_0
-    devices.torch_gc()
-
-    debug = "debug" in save_sets
-
-    if ("copy config" in save_sets) and ("(" not in result): extras.create_config(result.replace("Merged model saved in ",""), 0, informer(model_a), informer(model_b), informer(model_b))
-
-    if imggen :
-        images = simggen(s_prompt,s_nprompt,s_steps,s_sampler,s_cfg,s_seed,s_w,s_h,s_batch_size,
-                        genoptions,s_hrupscaler,s_hr2ndsteps,s_denois_str,s_hr_scale,
-                        currentmodel,id_sets,modelid,
-                        *txt2imgparams,debug = debug)
-
-        return result,currentmodel,*images[:4]
-    else:
-        return result,currentmodel
-
-def checkpointer_infomer(name):
-    return sd_models.get_closet_checkpoint_match(name)
-
-# XXX hack. fake checkpoint_info
-def fake_checkpoint_info(checkpoint_info,metadata,currentmodel):
-    from modules import cache
-    dump_cache = cache.dump_cache
-    c_cache = cache.cache
-    
-    checkpoint_info = deepcopy(checkpoint_info)
-    # change model name etc.
-    sha256 = hashlib.sha256(json.dumps(metadata).encode("utf-8")).hexdigest()
-    checkpoint_info.sha256 = sha256
-    checkpoint_info.name_for_extra = currentmodel
-
-    checkpoint_info.name = checkpoint_info.name_for_extra + ".safetensors"
-    checkpoint_info.model_name = checkpoint_info.name_for_extra.replace("/", "_").replace("\\", "_")
-    checkpoint_info.title = f"{checkpoint_info.name} [{sha256[0:10]}]"
-    checkpoint_info.metadata = metadata
-
-    # for sd-webui  v1.5.x
-    sd_models.checkpoints_list[checkpoint_info.title] = checkpoint_info
-
-        # force to set a new sha256 hash
-    if c_cache is not None: 
-        hashes = c_cache("hashes")
-        hashes[f"checkpoint/{checkpoint_info.name}"] = {
-        "mtime": os.path.getmtime(checkpoint_info.filename),
-        "sha256": sha256,
-        }
-        # save cache
-        dump_cache()
-
-    # set ids for a fake checkpoint info
-    checkpoint_info.ids = [checkpoint_info.model_name, checkpoint_info.name, checkpoint_info.name_for_extra]
-    return checkpoint_info
-
-NUM_INPUT_BLOCKS = 12
-NUM_MID_BLOCK = 1
-NUM_OUTPUT_BLOCKS = 12
-NUM_TOTAL_BLOCKS = NUM_INPUT_BLOCKS + NUM_MID_BLOCK + NUM_OUTPUT_BLOCKS
-BLOCKID=["BASE","IN00","IN01","IN02","IN03","IN04","IN05","IN06","IN07","IN08","IN09","IN10","IN11","M00","OUT00","OUT01","OUT02","OUT03","OUT04","OUT05","OUT06","OUT07","OUT08","OUT09","OUT10","OUT11"]
-BLOCKIDXLL=["BASE","IN00","IN01","IN02","IN03","IN04","IN05","IN06","IN07","IN08","M00","OUT00","OUT01","OUT02","OUT03","OUT04","OUT05","OUT06","OUT07","OUT08","VAE"]
-BLOCKIDXL=['BASE', 'IN0', 'IN1', 'IN2', 'IN3', 'IN4', 'IN5', 'IN6', 'IN7', 'IN8', 'M', 'OUT0', 'OUT1', 'OUT2', 'OUT3', 'OUT4', 'OUT5', 'OUT6', 'OUT7', 'OUT8', 'VAE']
-
-RANDMAP = [0,50,100] #alpha,beta,elements
-
-statistics = {"sum":{},"mean":{},"max":{},"min":{}}
-
-################################################
-##### Main Merging Code
-
-def smerge(weights_a,weights_b,model_a,model_b,model_c,base_alpha,base_beta,mode,calcmode,
-                useblocks,custom_name,save_sets,id_sets,wpresets,deep,fine,bake_in_vae,opt_value,inex,ex_blocks,ex_elems,deepprint,lucks,main = [False,False,False]):
-    
-    caster("merge start",hearm)
-    global hear,mergedmodel,stopmerge,statistics
-    stopmerge = False
-    
-    debug = "debug" in save_sets
-    uselerp = "use old calc method" not in save_sets
-    device = "cuda" if "use cuda" in save_sets else "cpu"
-
-    unload_model_weights(sd_models.model_data.sd_model)
-
-    # for from file
-    if type(useblocks) is str:
-        useblocks = True if useblocks =="True" else False
-    if type(base_alpha) == str:base_alpha = float(base_alpha)
-    if type(base_beta) == str:base_beta  = float(base_beta)
-
-    #random
-    if lucks != {}:
-        if lucks["seed"] == -1: lucks["ceed"] = str(random.randrange(4294967294))
-        else: lucks["ceed"] = lucks["seed"] 
-    else: lucks["ceed"]  = 0
-    np.random.seed(int(lucks["ceed"]))
-    randomer = np.random.rand(2500)
-
-    cachetarget =[]
-    for model,num in zip([model_a,model_b,model_c],main):
-        if model != "" and num:
-            cachetarget.append(model)
-
-    weights_a,deep = randdealer(weights_a,randomer,0,lucks,deep)
-    weights_b,_ = randdealer(weights_b,randomer,1,lucks,None)
-
-    weights_a_orig = weights_a
-    weights_b_orig = weights_b
-
-    # preset to weights
-    if wpresets != False and useblocks:
-        weights_a = wpreseter(weights_a,wpresets)
-        weights_b = wpreseter(weights_b,wpresets)
-
-    # mode select booleans
-    usebeta = MODES[2] in mode or MODES[3] in mode or "tensor" in calcmode
-    metadata = {"format": "pt"}
-
-    if (calcmode == "trainDifference" or calcmode == "extract") and "Add" not in mode:
-        print(f"{bcolors.WARNING}Mode changed to add difference{bcolors.ENDC}")
-        mode = "Add"
-    if model_c == "" or model_c is None:
-        #fallback to avoid crash
-        model_c = model_a
-        print(f"{bcolors.WARNING}Substituting empty model_c with model_a{bcolors.ENDC}")
-
-    if not useblocks:
-        weights_a = weights_b = ""
-    #for save log and save current model
-    mergedmodel =[weights_a,weights_b,
-                            hashfromname(model_a),hashfromname(model_b),hashfromname(model_c),
-                            base_alpha,base_beta,mode,useblocks,custom_name,save_sets,id_sets,deep,calcmode,lucks["ceed"],fine,opt_value,inex,ex_blocks,ex_elems].copy()
-
-    model_a = namefromhash(model_a)
-    model_b = namefromhash(model_b)
-    model_c = namefromhash(model_c)
-
-    caster(mergedmodel,False)
-
-    #elementals
-    if len(deep) > 0:
-        deep = deep.replace("\n",",")
-        deep = deep.replace(calcmode+",","")
-        deep = deep.split(",")
-
-    #format check
-    if model_a =="" or model_b =="" or ((not MODES[0] in mode) and model_c=="") : 
-        return "ERROR: Necessary model is not selected",*NON4
-    
-    #for MBW text to list
-    if useblocks:
-        weights_a_t=weights_a.split(',',1)
-        weights_b_t=weights_b.split(',',1)
-        base_alpha  = float(weights_a_t[0])    
-        weights_a = [float(w) for w in weights_a_t[1].split(',')]
-        caster(f"from {weights_a_t}, alpha = {base_alpha},weights_a ={weights_a}",hearm)
-        if not (len(weights_a) == 25 or len(weights_a) == 19):return f"ERROR: weights alpha value must be 20 or 26.",*NON4
-        if usebeta:
-            base_beta = float(weights_b_t[0]) 
-            weights_b = [float(w) for w in weights_b_t[1].split(',')]
-            caster(f"from {weights_b_t}, beta = {base_beta},weights_a ={weights_b}",hearm)
-            if not(len(weights_b) == 25 or len(weights_b) == 19): return f"ERROR: weights beta value must be 20 or 26.",*NON4
-        
-    caster("model load start",hearm)
-    printstart(model_a,model_b,model_c,base_alpha,base_beta,weights_a,weights_b,mode,useblocks,calcmode,deep,lucks['ceed'],fine,inex,ex_blocks,ex_elems)
-
-    theta_1=load_model_weights_m(model_b,2,cachetarget,device).copy()
-
-    isxl = "conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj.weight" in theta_1.keys()
-
-    #adjust
-    if fine.rstrip(",0") != "":
-        fine = fineman(fine,isxl)
-    else:
-        fine = ""
-
-    if isxl and useblocks:
-        if len(weights_a) == 25:
-            weights_a = weighttoxl(weights_a)
-            print(f"weight converted for XL{weights_a}")
-        if usebeta:
-            if len(weights_b) == 25:
-                weights_b = weighttoxl(weights_b)
-                print(f"weight converted for XL{weights_b}")
-        if len(weights_a) == 19: weights_a = weights_a + [0]
-        if len(weights_b) == 19: weights_b = weights_b + [0]
-
-    if MODES[1] in mode:#Add
-        if stopmerge: return "STOPPED", *NON4
-        if calcmode == "trainDifference" or calcmode == "extract":
-            theta_2 = load_model_weights_m(model_c,3,cachetarget,device).copy()
-        else:
-            theta_2 = load_model_weights_m(model_c,3,cachetarget,device).copy()
-            for key in tqdm(theta_1.keys()):
-                if 'model' in key:
-                    if key in theta_2:
-                        t2 = theta_2.get(key, torch.zeros_like(theta_1[key]))
-                        theta_1[key] = theta_1[key]- t2
-                    else:
-                        theta_1[key] = torch.zeros_like(theta_1[key])
-            del theta_2
-
-    if stopmerge: return "STOPPED", *NON4
-    
-    if  "tensor" in calcmode or "self" in calcmode:
-        theta_t = load_model_weights_m(model_a,1,cachetarget,device).copy()
-        theta_0 ={}
-        for key in theta_t:
-            theta_0[key] = theta_t[key].clone()
-        del theta_t
-    else:
-        theta_0=load_model_weights_m(model_a,1,cachetarget,device).copy()
-
-    if MODES[2] in mode or MODES[3] in mode:#Tripe or Twice
-        theta_2 = load_model_weights_m(model_c,3,cachetarget,device).copy()
-    else:
-        if not (calcmode == "trainDifference" or calcmode == "extract"):
-            theta_2 = {}
-
-    alpha = base_alpha
-    beta = base_beta
-
-    ex_elems = ex_elems.split(",")
-
-    keyratio = []
-    key_and_alpha = {}
-
-    ##### Stage 0/2 in Cosine
-    if "cosine" in calcmode:
-        sim, sims = precosine("A" in calcmode,theta_0,theta_1)
-
-    ##### Stage 1/2
-
-    for num, key in enumerate(tqdm(theta_0.keys(), desc="Stage 1/2") if not False else theta_0.keys()):
-        if stopmerge: return "STOPPED", *NON4
-        if not ("model" in key and key in theta_1): continue
-        if not ("weight" in key or "bias" in key): continue
-        if calcmode == "trainDifference" or calcmode == "extract":
-            if key not in theta_2:
-                continue
-        else:
-            if usebeta and (not key in theta_2) and (not theta_2 == {}) :
-                continue
-
-        weight_index = -1
-        current_alpha = alpha
-        current_beta = beta
-
-        a = list(theta_0[key].shape)
-        b = list(theta_1[key].shape)
-
-        # this enables merging an inpainting model (A) with another one (B);
-        # where normal model would have 4 channels, for latenst space, inpainting model would
-        # have another 4 channels for unmasked picture's latent space, plus one channel for mask, for a total of 9
-        if a != b and a[0:1] + a[2:] == b[0:1] + b[2:]:
-            if a[1] == 4 and b[1] == 9:
-                raise RuntimeError("When merging inpainting model with a normal one, A must be the inpainting model.")
-            if a[1] == 4 and b[1] == 8:
-                raise RuntimeError("When merging instruct-pix2pix model with a normal one, A must be the instruct-pix2pix model.")
-
-            if a[1] == 8 and b[1] == 4:#If we have an Instruct-Pix2Pix model...
-                result_is_instruct_pix2pix_model = True
-            else:
-                assert a.shape[1] == 9 and b.shape[1] == 4, f"Bad dimensions for merged layer {key}: A={a.shape}, B={b.shape}"
-                result_is_inpainting_model = True
-
-        block,blocks26 = blockfromkey(key,isxl)
-        if block == "Not Merge": continue
-        if inex != "Off" and (ex_blocks or (ex_elems != [""])) and excluder(blocks26,inex,ex_blocks,ex_elems,key): continue
-        weight_index = BLOCKIDXLL.index(blocks26) if isxl else BLOCKID.index(blocks26)
-
-        if useblocks:
-            if weight_index > 0: 
-                current_alpha = weights_a[weight_index - 1] 
-                if usebeta:
-                    current_beta = weights_b[weight_index - 1] 
-
-        if len(deep) > 0:
-            current_alpha = elementals(key,weight_index,deep,randomer,num,lucks,deepprint,current_alpha)
-
-        keyratio.append([key,current_alpha, current_beta])
-        #keyratio.append([key,current_alpha, current_beta,list(theta_0[key].shape),torch.sum(theta_0[key]).item(), torch.mean(theta_0[key]).item(), torch.max(theta_0[key]).item(),  torch.min(theta_0[key]).item()])
-
-        if calcmode == "normal":
-            if a != b and a[0:1] + a[2:] == b[0:1] + b[2:]:
-                # Merge only the vectors the models have in common.  Otherwise we get an error due to dimension mismatch.
-                theta_0_a = theta_0[key][:, 0:4, :, :]
-            else:
-                theta_0_a = theta_0[key]
-
-            if MODES[1] in mode:#Add
-                caster(f"{num}, {block}, {model_a}+{current_alpha}+*({model_b}-{model_c}),{key}",hear)
-                theta_0_a = theta_0_a + current_alpha * theta_1[key]
-                
-            elif MODES[2] in mode:#Triple
-                caster(f"{num}, {block}, {model_a}+{1-current_alpha-current_beta}+{model_b}*{current_alpha}+ {model_c}*{current_beta}",hear)
-                #
-                if uselerp and current_alpha + current_beta != 0:
-                    theta_0_a =lerp(theta_0_a.to(torch.float32),lerp(theta_1[key].to(torch.float32),theta_2[key].to(torch.float32),current_beta/(current_alpha + current_beta)),current_alpha + current_beta).to(theta_0_a.dtype)
-                else:
-                    theta_0_a = (1 - current_alpha-current_beta) * theta_0_a + current_alpha * theta_1[key]+current_beta * theta_2[key] 
-
-            elif MODES[3] in mode:#Twice
-                caster(f"{num}, {block}, {key},{model_a} +  {1-current_alpha} + {model_b}*{current_alpha}",hear)
-                caster(f"{num}, {block}, {key}({model_a}+{model_b}) +{1-current_beta}+{model_c}*{current_beta}",hear)
-                if uselerp:
-                    theta_0_a = torch.lerp(torch.lerp(theta_0_a.to(torch.float32), theta_1[key].to(torch.float32), current_alpha), theta_2[key].to(torch.float32), current_beta).to(theta_0_a.dtype)
-                else:
-                    theta_0_a = (1 - current_alpha) * theta_0_a + current_alpha * theta_1[key]
-                    theta_0_a = (1 - current_beta) * theta_0_a + current_beta * theta_2[key]
-
-            else:#Weight
-                if current_alpha == 1:
-                    caster(f"{num}, {block}, {key} alpha = 1,{model_a}={model_b}",hear)
-                    theta_0_a = theta_1[key]
-                elif current_alpha !=0:
-                    caster(f"{num}, {block}, {key}, {model_a}*{1-current_alpha}+{model_b}*{current_alpha}",hear)
-                    if uselerp:
-                        theta_0_a = torch.lerp(theta_0_a.to(torch.float32), theta_1[key].to(torch.float32), current_alpha).to(theta_0_a.dtype)
-                    else:
-                        theta_0_a = (1 - current_alpha) * theta_0_a + current_alpha * theta_1[key]
-
-            if a != b and a[0:1] + a[2:] == b[0:1] + b[2:]:
-                theta_0[key][:, 0:4, :, :] = theta_0_a
-            else:
-                theta_0[key] = theta_0_a
-            
-            del theta_0_a, a, b
-
-        elif "cosine" in calcmode:
-            if "first_stage_model" in key: continue
-            cosine(calcmode,key,sim,sims,current_alpha,theta_0,theta_1,num,block,uselerp)
-
-        elif calcmode == "trainDifference":
-            if torch.allclose(theta_1[key].float(), theta_2[key].float(), rtol=0, atol=0):
-                theta_2[key] = theta_0[key]
-                continue
-            traindiff(key,current_alpha,theta_0,theta_1,theta_2)
-
-        elif calcmode == "smoothAdd":
-            caster(f"{num}, {block}, model A[{key}] +  {current_alpha} + * (model B - model C)[{key}]", hear)
-            # Apply median filter to the weight differences
-            filtered_diff = scipy.ndimage.median_filter(theta_1[key].to(torch.float32).cpu().numpy(), size=3)
-            # Apply Gaussian filter to the filtered differences
-            filtered_diff = scipy.ndimage.gaussian_filter(filtered_diff, sigma=1)
-            theta_1[key] = torch.tensor(filtered_diff)
-            # Add the filtered differences to the original weights
-            theta_0[key] = theta_0[key] + current_alpha * theta_1[key]
-
-        elif calcmode == "smoothAdd MT":
-            key_and_alpha[key] = current_alpha
-
-        elif "tensor" in calcmode:
-            dim = theta_0[key].dim()
-            if dim == 0 : continue
-            tensormerge("2" not in calcmode,key,dim,theta_0,theta_1,current_alpha,current_beta)
-
-        elif "extract" == calcmode:
-            theta_0[key] = extract_super(theta_0[key],theta_1[key],theta_2[key],current_alpha,current_beta,opt_value)
-
-        elif calcmode == "self":
-            if any(selfkey in key for selfkey in SELFKEYS):continue
-            if current_alpha == 0: continue
-            theta_0[key] = (theta_0[key].clone()) * current_alpha
-
-        elif calcmode == "plus random":
-            if any(selfkey in key for selfkey in SELFKEYS):continue
-            if current_alpha == 0: continue
-            theta_0[key] +=  torch.randn_like(theta_0[key].clone()) * current_alpha
-
-        ##### Adjust
-        if any(item in key for item in FINETUNES) and fine:
-            index = FINETUNES.index(key)
-            if 5 > index : 
-                theta_0[key] =theta_0[key]* fine[index] 
-            else :theta_0[key] =theta_0[key] + torch.tensor(fine[5]).to(theta_0[key].device)
-
-
-    if calcmode == "smoothAdd MT":
-        # setting threads to higher than 8 doesn't significantly affect the time for merging
-        threads = cpu_count()
-        tasks_per_thread = 8
-
-        theta_0, theta_1, stopped = multithread_smoothadd(key_and_alpha, theta_0, theta_1, threads, tasks_per_thread, hear)
-        if stopped:
-            return "STOPPED", *NON4
-
-    currentmodel = makemodelname(weights_a,weights_b,model_a, model_b,model_c, base_alpha,base_beta,useblocks,mode,calcmode)
-
-    for key in tqdm(theta_1.keys(), desc="Stage 2/2"):
-        if key in CHCKPOINT_DICT_SKIP_ON_MERGE:
-            continue
-        if "model" in key and key not in theta_0:
-            theta_0.update({key:theta_1[key]})
-
-    del theta_1
-    if calcmode == "trainDifference" or calcmode == "extract":
-        del theta_2
-
-    ##### BakeVAE
-    bake_in_vae_filename = sd_vae.vae_dict.get(bake_in_vae, None)
-    if bake_in_vae_filename is not None:
-        print(f"Baking in VAE from {bake_in_vae_filename}")
-        vae_dict = sd_vae.load_vae_dict(bake_in_vae_filename, map_location='cpu')
-
-        for key in vae_dict.keys():
-            theta_0_key = 'first_stage_model.' + key
-            if theta_0_key in theta_0:
-                theta_0[theta_0_key] = vae_dict[key]
-
-        del vae_dict
-
-    modelid = rwmergelog(currentmodel,mergedmodel)
-    if "save E-list" in lucks["set"]: saveekeys(keyratio,modelid)
-
-    caster(mergedmodel,False)
-    if "Reset CLIP ids" in save_sets: resetclip(theta_0)
-
-    if True: # always set metadata. savemodel() will check save_sets later
-        merge_recipe = {
-            "type": "sd-webui-supermerger",
-            "weights_alpha": weights_a if useblocks else None,
-            "weights_beta": weights_b if useblocks else None,
-            "weights_alpha_orig": weights_a_orig if useblocks else None,
-            "weights_beta_orig": weights_b_orig if useblocks else None,
-            "model_a": longhashfromname(model_a),
-            "model_b": longhashfromname(model_b),
-            "model_c": longhashfromname(model_c),
-            "base_alpha": base_alpha,
-            "base_beta": base_beta,
-            "mode": mode,
-            "mbw": useblocks,
-            "elemental_merge": deep,
-            "calcmode" : calcmode,
-            f"{inex}":ex_blocks + ex_elems
-            }
-        metadata["sd_merge_recipe"] = json.dumps(merge_recipe)
-        metadata["sd_merge_models"] = {}
-
-        def add_model_metadata(checkpoint_name):
-            checkpoint_info = sd_models.get_closet_checkpoint_match(checkpoint_name)
-            checkpoint_info.calculate_shorthash()
-            metadata["sd_merge_models"][checkpoint_info.sha256] = {
-                "name": checkpoint_name,
-                "legacy_hash": checkpoint_info.hash
-            }
-
-            #metadata["sd_merge_models"].update(checkpoint_info.metadata.get("sd_merge_models", {}))
-
-        if model_a:
-            add_model_metadata(model_a)
-        if model_b:
-            add_model_metadata(model_b)
-        if model_c:
-            add_model_metadata(model_c)
-
-        metadata["sd_merge_models"] = json.dumps(metadata["sd_merge_models"])
-
-    return "",currentmodel,modelid,theta_0,metadata
-
-################################################
-##### cosineA/B
-def precosine(calcmode,theta_0,theta_1):
-    if calcmode: #favors modelA's structure with details from B
-        if stopmerge: return "STOPPED", *NON4
-        sim = torch.nn.CosineSimilarity(dim=0)
-        sims = np.array([], dtype=np.float64)
-        for key in (tqdm(theta_0.keys(), desc="Stage 0/2")):
-            # skip VAE model parameters to get better results
-            if "first_stage_model" in key: continue
-            if "model" in key and key in theta_1:
-                theta_0_norm = nn.functional.normalize(theta_0[key].to(torch.float32), p=2, dim=0)
-                theta_1_norm = nn.functional.normalize(theta_1[key].to(torch.float32), p=2, dim=0)
-                simab = sim(theta_0_norm, theta_1_norm)
-                sims = np.append(sims,simab.cpu().numpy())
-        sims = sims[~np.isnan(sims)]
-        sims = np.delete(sims, np.where(sims<np.percentile(sims, 1 ,method = 'midpoint')))
-        sims = np.delete(sims, np.where(sims>np.percentile(sims, 99 ,method = 'midpoint')))
-    else: #favors modelB's structure with details from A
-        if stopmerge: return "STOPPED", *NON4
-        sim = torch.nn.CosineSimilarity(dim=0)
-        sims = np.array([], dtype=np.float64)
-        for key in (tqdm(theta_0.keys(), desc="Stage 0/2")):
-            # skip VAE model parameters to get better results
-            if "first_stage_model" in key: continue
-            if "model" in key and key in theta_1:
-                simab = sim(theta_0[key].to(torch.float32), theta_1[key].to(torch.float32))
-                dot_product = torch.dot(theta_0[key].view(-1).to(torch.float32), theta_1[key].view(-1).to(torch.float32))
-                magnitude_similarity = dot_product / (torch.norm(theta_0[key].to(torch.float32)) * torch.norm(theta_1[key].to(torch.float32)))
-                combined_similarity = (simab + magnitude_similarity) / 2.0
-                sims = np.append(sims, combined_similarity.cpu().numpy())
-        sims = sims[~np.isnan(sims)]
-        sims = np.delete(sims, np.where(sims < np.percentile(sims, 1, method='midpoint')))
-        sims = np.delete(sims, np.where(sims > np.percentile(sims, 99, method='midpoint')))
-    return sim, sims
-
-def cosine(mode,key,sim,sims,current_alpha,theta_0,theta_1,num,block,uselerp):
-    if "A" in mode: #favors modelA's structure with details from B
-        # skip VAE model parameters to get better results
-        if "model" in key and key in theta_0:
-            # Normalize the vectors before merging
-            theta_0_norm = nn.functional.normalize(theta_0[key].to(torch.float32), p=2, dim=0)
-            theta_1_norm = nn.functional.normalize(theta_1[key].to(torch.float32), p=2, dim=0)
-            simab = sim(theta_0_norm, theta_1_norm)
-            dot_product = torch.dot(theta_0_norm.view(-1), theta_1_norm.view(-1))
-            magnitude_similarity = dot_product / (torch.norm(theta_0_norm) * torch.norm(theta_1_norm))
-            combined_similarity = (simab + magnitude_similarity) / 2.0
-            k = (combined_similarity - sims.min()) / (sims.max() - sims.min())
-            k = k - abs(current_alpha)
-            k = k.clip(min=0,max=1.0)
-            caster(f"{num}, {block}, model A[{key}] {1-k} +  (model B)[{key}]*{k}",hear)
-            if uselerp:
-                theta_0[key] = lerp(theta_1[key].to(torch.float32), theta_0[key].to(torch.float32),k).to(theta_0[key].dtype)
-            else:
-                theta_0[key] = theta_1[key] * (1 - k) + theta_0[key] * k
-
-    else: #favors modelB's structure with details from A
-        # skip VAE model parameters to get better results
-        if "model" in key and key in theta_0:
-            simab = sim(theta_0[key].to(torch.float32), theta_1[key].to(torch.float32))
-            dot_product = torch.dot(theta_0[key].view(-1).to(torch.float32), theta_1[key].view(-1).to(torch.float32))
-            magnitude_similarity = dot_product / (torch.norm(theta_0[key].to(torch.float32)) * torch.norm(theta_1[key].to(torch.float32)))
-            combined_similarity = (simab + magnitude_similarity) / 2.0
-            k = (combined_similarity - sims.min()) / (sims.max() - sims.min())
-            k = k - current_alpha
-            k = k.clip(min=0,max=1.0)
-            caster(f"{num}, {block}, model A[{key}] *{1-k} + (model B)[{key}]*{k}",hear)
-            if uselerp:
-                theta_0[key] = lerp(theta_1[key].to(torch.float32), theta_0[key].to(torch.float32),k).to(theta_0[key].dtype)
-            else:
-                theta_0[key] = theta_1[key] * (1 - k) + theta_0[key] * k
-
-################################################
-##### Traindiff
-def traindiff(key,current_alpha,theta_0,theta_1,theta_2):
-            # Check if theta_1[key] is equal to theta_2[key]
-    diff_AB = theta_1[key].float() - theta_2[key].float()
-
-    distance_A0 = torch.abs(theta_1[key].float() - theta_2[key].float())
-    distance_A1 = torch.abs(theta_1[key].float() - theta_0[key].float())
-
-    sum_distances = distance_A0 + distance_A1
-
-    scale = torch.where(sum_distances != 0, distance_A1 / sum_distances, torch.tensor(0.).float())
-    sign_scale = torch.sign(theta_1[key].float() - theta_2[key].float())
-    scale = sign_scale * torch.abs(scale)
-
-    new_diff = scale * torch.abs(diff_AB)
-    theta_0[key] = theta_0[key] + (new_diff * (current_alpha*1.8))
-
-################################################
-##### Extract
-# fix for python 3.9.16
-from typing import Union, Optional
-def extract_super(base: Optional[Tensor], a: Tensor, b: Tensor, alpha: float, beta: float, gamma: float) -> Tensor:
-# def extract_super(base: Tensor | None, a: Tensor, b: Tensor, alpha: float, beta: float, gamma: float) -> Tensor:
-    assert base is None or base.shape == a.shape
-    assert a.shape == b.shape
-    assert 0 <= alpha <= 1
-    assert 0 <= beta <= 1
-    assert 0 <= gamma
-    dtype = base.dtype if base is not None else a.dtype
-    base = base.float() if base is not None else 0
-    a = a.float() - base
-    b = b.float() - base
-    c = cosine_similarity(a, b, -1).clamp(-1, 1).unsqueeze(-1)
-    d = ((c + 1) / 2) ** gamma
-    result = base + lerp(a, b, alpha) * lerp(d, 1 - d, beta)
-    return result.to(dtype)
-
-def extract(a: Tensor, b: Tensor, p: float, smoothness: float) -> Tensor:
-    assert a.shape == b.shape
-    assert 0 <= p <= 1
-    assert 0 <= smoothness <= 1
-    
-    r = relu if smoothness == 0 else partial(softplus, beta=1 / smoothness)
-    c = r(cosine_similarity(a, b, dim=-1)).unsqueeze(dim=-1).repeat_interleave(b.shape[-1], -1)
-    m = torch.lerp(c, torch.ones_like(c) - c, p)
-    return a * m
-
-################################################
-##### Tensor Merge
-def tensormerge(mode,key,dim, theta_0,theta_1,current_alpha,current_beta):
-    if mode:
-        if current_alpha+current_beta <= 1 :
-            talphas = int(theta_0[key].shape[0]*(current_beta))
-            talphae = int(theta_0[key].shape[0]*(current_alpha+current_beta))
-            if dim == 1:
-                theta_0[key][talphas:talphae] = theta_1[key][talphas:talphae].clone()
-
-            elif dim == 2:
-                theta_0[key][talphas:talphae,:] = theta_1[key][talphas:talphae,:].clone()
-
-            elif dim == 3:
-                theta_0[key][talphas:talphae,:,:] = theta_1[key][talphas:talphae,:,:].clone()
-
-            elif dim == 4:
-                theta_0[key][talphas:talphae,:,:,:] = theta_1[key][talphas:talphae,:,:,:].clone()
-
-        else:
-            talphas = int(theta_0[key].shape[0]*(current_alpha+current_beta-1))
-            talphae = int(theta_0[key].shape[0]*(current_beta))
-            theta_t = theta_1[key].clone()
-            if dim == 1:
-                theta_t[talphas:talphae] = theta_0[key][talphas:talphae].clone()
-
-            elif dim == 2:
-                theta_t[talphas:talphae,:] = theta_0[key][talphas:talphae,:].clone()
-
-            elif dim == 3:
-                theta_t[talphas:talphae,:,:] = theta_0[key][talphas:talphae,:,:].clone()
-
-            elif dim == 4:
-                theta_t[talphas:talphae,:,:,:] = theta_0[key][talphas:talphae,:,:,:].clone()
-            theta_0[key] = theta_t
-
-    else:
-        if current_alpha+current_beta <= 1 :
-            talphas = int(theta_0[key].shape[0]*(current_beta))
-            talphae = int(theta_0[key].shape[0]*(current_alpha+current_beta))
-            if dim > 1:
-                if theta_0[key].shape[1] > 100:
-                    talphas = int(theta_0[key].shape[1]*(current_beta))
-                    talphae = int(theta_0[key].shape[1]*(current_alpha+current_beta))
-            if dim == 1:
-                theta_0[key][talphas:talphae] = theta_1[key][talphas:talphae].clone()
-
-            elif dim == 2:
-                theta_0[key][:,talphas:talphae] = theta_1[key][:,talphas:talphae].clone()
-
-            elif dim == 3:
-                theta_0[key][:,talphas:talphae,:] = theta_1[key][:,talphas:talphae,:].clone()
-
-            elif dim == 4:
-                theta_0[key][:,talphas:talphae,:,:] = theta_1[key][:,talphas:talphae,:,:].clone()
-
-        else:
-            talphas = int(theta_0[key].shape[0]*(current_alpha+current_beta-1))
-            talphae = int(theta_0[key].shape[0]*(current_beta))
-            theta_t = theta_1[key].clone()
-            if dim > 1:
-                if theta_0[key].shape[1] > 100:
-                    talphas = int(theta_0[key].shape[1]*(current_alpha+current_beta-1))
-                    talphae = int(theta_0[key].shape[1]*(current_beta))
-            if dim == 1:
-                theta_t[talphas:talphae] = theta_0[key][talphas:talphae].clone()
-
-            elif dim == 2:
-                theta_t[:,talphas:talphae] = theta_0[key][:,talphas:talphae].clone()
-
-            elif dim == 3:
-                theta_t[:,talphas:talphae,:] = theta_0[key][:,talphas:talphae,:].clone()
-
-            elif dim == 4:
-                theta_t[:,talphas:talphae,:,:] = theta_0[key][:,talphas:talphae,:,:].clone()
-            theta_0[key] = theta_t
-
-################################################
-##### Multi Thread SmoothAdd
-
-def multithread_smoothadd(key_and_alpha, theta_0, theta_1, threads, tasks_per_thread, hear):  
-    lock_theta_0 = Lock()
-    lock_theta_1 = Lock()
-    lock_progress = Lock()
-
-    def thread_callback(keys):
-        nonlocal theta_0, theta_1
-        if stopmerge:
-            return False
-
-        for key in keys:
-            caster(f"model A[{key}] +  {key_and_alpha[key]} + * (model B - model C)[{key}]", hear)
-            filtered_diff = scipy.ndimage.median_filter(theta_1[key].to(torch.float32).cpu().numpy(), size=3)
-            filtered_diff = scipy.ndimage.gaussian_filter(filtered_diff, sigma=1)
-            with lock_theta_1:
-                theta_1[key] = torch.tensor(filtered_diff)
-            with lock_theta_0:
-                theta_0[key] = theta_0[key] + key_and_alpha[key] * theta_1[key]
-
-        with lock_progress:
-            progress.update(len(keys))
-
-        return True
-
-    def extract_and_remove(input_list, count):
-        extracted = input_list[:count]
-        del input_list[:count]
-
-        return extracted
-
-    keys = list(key_and_alpha.keys())
-
-    total_threads = ceil(len(keys) / int(tasks_per_thread))
-    print(f"max threads = {threads}, total threads = {total_threads}, tasks per thread = {tasks_per_thread}")
-
-    progress = tqdm(key_and_alpha.keys(), desc="smoothAdd MT")
-
-    futures = []
-    with ThreadPoolExecutor(max_workers=threads) as executor:
-        futures = [executor.submit(thread_callback, extract_and_remove(keys, int(tasks_per_thread))) for i in range(total_threads)]
-        for future in as_completed(futures):
-            if not future.result():
-                executor.shutdown()
-                return theta_0, theta_1, True
-        del progress
-
-    return theta_0, theta_1, False
-
-################################################
-##### Elementals
-def elementals(key,weight_index,deep,randomer,num,lucks,deepprint,current_alpha):
-    skey = key + BLOCKID[weight_index]
-    for d in deep:
-        if d.count(":") != 2 :continue
-        dbs,dws,dr = d.split(":")[0],d.split(":")[1],d.split(":")[2]
-        dbs = blocker(dbs,BLOCKID)
-        dbs,dws = dbs.split(" "), dws.split(" ")
-        dbn,dbs = (True,dbs[1:]) if dbs[0] == "NOT" else (False,dbs)
-        dwn,dws = (True,dws[1:]) if dws[0] == "NOT" else (False,dws)
-        flag = dbn
-        for db in dbs:
-            if db in skey:
-                flag = not dbn
-        if flag:flag = dwn
-        else:continue
-        for dw in dws:
-            if dw in skey:
-                flag = not dwn
-        if flag:
-            dr = eratiodealer(dr,randomer,weight_index,num,lucks)
-            if deepprint :print(" ", dbs,dws,key,dr)
-            current_alpha = dr
-    return current_alpha
-
-def forkforker(filename,device):
-    try:
-        return sd_models.read_state_dict(filename,map_location = device)
-    except:
-        return sd_models.read_state_dict(filename)
-
-################################################
-##### Load Model
-
-def load_model_weights_m(model,abc,cachetarget,device):
-    checkpoint_info = sd_models.get_closet_checkpoint_match(model)
-    sd_model_name = checkpoint_info.model_name
-
-    if checkpoint_info in modelcache:
-        print(f"Loading weights [{sd_model_name}] from cache")
-        return {k: v.to(device) for k, v in modelcache[checkpoint_info].items()}
-    else:
-        print(f"Loading weights [{sd_model_name}] from file")
-        state_dict = forkforker(checkpoint_info.filename,device)
-        if orig_cache >= abc:
-            modelcache[checkpoint_info] = state_dict
-            modelcache[checkpoint_info] = {k: v.to("cpu") for k, v in modelcache[checkpoint_info].items()}
-        dontdelete = []
-        for model in cachetarget:
-            dontdelete.append(sd_models.get_closet_checkpoint_match(model))
-        while len(modelcache) > orig_cache:
-            for key in modelcache.keys():
-                if key in dontdelete:continue
-                modelcache.pop(key)
-                break
-        return state_dict
-
-def makemodelname(weights_a,weights_b,model_a, model_b,model_c, alpha,beta,useblocks,mode,calc):
-    model_a=filenamecutter(model_a)
-    model_b=filenamecutter(model_b)
-    model_c=filenamecutter(model_c)
-
-    if type(alpha) == str:alpha = float(alpha)
-    if type(beta)== str:beta  = float(beta)
-
-    if useblocks:
-        if MODES[1] in mode:#add
-            currentmodel =f"{model_a} + ({model_b} - {model_c}) x alpha ({str(round(alpha,3))},{','.join(str(s) for s in weights_a)})"
-        elif MODES[2] in mode:#triple
-            currentmodel =f"{model_a} x (1-alpha-beta) + {model_b} x alpha + {model_c} x beta (alpha = {str(round(alpha,3))},{','.join(str(s) for s in weights_a)},beta = {beta},{','.join(str(s) for s in weights_b)})"
-        elif MODES[3] in mode:#twice
-            currentmodel =f"({model_a} x (1-alpha) + {model_b} x alpha)x(1-beta)+  {model_c} x beta ({str(round(alpha,3))},{','.join(str(s) for s in weights_a)})_({str(round(beta,3))},{','.join(str(s) for s in weights_b)})"
-        else:
-            currentmodel =f"{model_a} x (1-alpha) + {model_b} x alpha ({str(round(alpha,3))},{','.join(str(s) for s in weights_a)})"
-    else:
-        if MODES[1] in mode:#add
-            currentmodel =f"{model_a} + ({model_b} -  {model_c}) x {str(round(alpha,3))}"
-        elif MODES[2] in mode:#triple
-            currentmodel =f"{model_a} x {str(round(1-alpha-beta,3))} + {model_b} x {str(round(alpha,3))} + {model_c} x {str(round(beta,3))}"
-        elif MODES[3] in mode:#twice
-            currentmodel =f"({model_a} x {str(round(1-alpha,3))} +{model_b} x {str(round(alpha,3))}) x {str(round(1-beta,3))} + {model_c} x {str(round(beta,3))}"
-        else:
-            currentmodel =f"{model_a} x {str(round(1-alpha,3))} + {model_b} x {str(round(alpha,3))}"
-    if calc != "normal":
-        currentmodel = currentmodel + "_" + calc
-        if calc == "tensor":
-            currentmodel = currentmodel + f"_beta_{beta}"
-    return currentmodel
-
-path_root = scripts.basedir()
-
-
-################################################
-##### Logging
-
-def rwmergelog(mergedname = "",settings= [],id = 0):
-    # for compatible
-    mode_info = {
-        "Weight sum": "Weight sum:A*(1-alpha)+B*alpha",
-        "Add difference": "Add difference:A+(B-C)*alpha",
-        "Triple sum": "Triple sum:A*(1-alpha-beta)+B*alpha+C*beta",
-        "sum Twice": "sum Twice:(A*(1-alpha)+B*alpha)*(1-beta)+C*beta",
-    }
-    setting = settings.copy()
-    if len(setting) > 7 and setting[7] in mode_info:
-        setting[7] = mode_info[setting[7]] # fix mode entry for compatible
-    filepath = os.path.join(path_root, "mergehistory.csv")
-    is_file = os.path.isfile(filepath)
-    if not is_file:
-        with open(filepath, 'a') as f:
-                                       #msettings=[0 weights_a,1 weights_b,2 model_a,3 model_b,4 model_c,5 base_alpha,6 base_beta,7 mode,8 useblocks,9 custom_name,10 save_sets,11 id_sets, 12 deep 13 calcmode]
-            f.writelines('"ID","time","name","weights alpha","weights beta","model A","model B","model C","alpha","beta","mode","use MBW","plus lora","custum name","save setting","use ID"\n')
-    with  open(filepath, 'r+') as f:
-        reader = csv.reader(f)
-        mlist = [raw for raw in reader]
-        if mergedname != "":
-            mergeid = len(mlist)
-            setting.insert(0,mergedname)
-            for i,x in enumerate(setting):
-                if "," in str(x) or "\n" in str(x):setting[i] = f'"{str(setting[i])}"'
-            text = ",".join(map(str, setting))
-            text=str(mergeid)+","+datetime.datetime.now().strftime('%Y.%m.%d %H.%M.%S.%f')[:-7]+"," + text + "\n"
-            f.writelines(text)
-            return mergeid
-        try:
-            out = mlist[int(id)]
-        except:
-            out = "ERROR: OUT of ID index"
-        return out
-
-def saveekeys(keyratio,modelid):
-    import csv
-    path_root = scripts.basedir()
-    dir_path = os.path.join(path_root,"extensions","sd-webui-supermerger","scripts", "data")
-
-    if not os.path.exists(dir_path):
-        os.makedirs(dir_path)
-    
-    filepath = os.path.join(dir_path,f"{modelid}.csv")
-
-    with open(filepath, 'w', newline='') as csvfile:
-        writer = csv.writer(csvfile)
-        writer.writerows(keyratio)
-
-def savestatics(modelid):
-    for key in statistics.keys():
-        result = [[tkey] + list(statistics[key][tkey]) for tkey in statistics[key].keys()]
-        saveekeys(result,f"{modelid}_{key}")
-
-def get_font(fontsize):
-    fontpath = os.path.join(scriptpath, "Roboto-Regular.ttf")
-    try:
-        return ImageFont.truetype(opts.font or fontpath, fontsize)
-    except Exception:
-        return ImageFont.truetype(fontpath, fontsize)
-
-def draw_origin(grid, text,width,height,width_one):
-    grid_d= Image.new("RGB", (grid.width,grid.height), "white")
-    grid_d.paste(grid,(0,0))
-
-    d= ImageDraw.Draw(grid_d)
-    color_active = (0, 0, 0)
-    fontsize = (width+height)//25
-    fnt = get_font(fontsize)
-
-    if grid.width != width_one:
-        while d.multiline_textsize(text, font=fnt)[0] > width_one*0.75 and fontsize > 0:
-            fontsize -=1
-            fnt = get_font(fontsize)
-    d.multiline_text((0,0), text, font=fnt, fill=color_active,align="center")
-    return grid_d
-
-def wpreseter(w,presets):
-    if "," not in w and w != "":
-        presets=presets.splitlines()
-        wdict={}
-        for l in presets:
-            if ":" in l :
-                key = l.split(":",1)[0]
-                wdict[key.strip()]=l.split(":",1)[1]
-            if "\t" in l:
-                key = l.split("\t",1)[0]
-                wdict[key.strip()]=l.split("\t",1)[1]
-        if w.strip() in wdict:
-            name = w
-            w = wdict[w.strip()]
-            print(f"weights {name} imported from presets : {w}")
-    return w
-
-def fullpathfromname(name):
-    if hash == "" or hash ==[]: return ""
-    checkpoint_info = sd_models.get_closet_checkpoint_match(name)
-    return checkpoint_info.filename
-
-def namefromhash(hash):
-    if hash == "" or hash ==[]: return ""
-    checkpoint_info = sd_models.get_closet_checkpoint_match(hash)
-    return checkpoint_info.model_name
-
-def hashfromname(name):
-    from modules import sd_models
-    if name == "" or name ==[]: return ""
-    checkpoint_info = sd_models.get_closet_checkpoint_match(name)
-    if checkpoint_info.shorthash is not None:
-        return checkpoint_info.shorthash
-    return checkpoint_info.calculate_shorthash()
-
-def longhashfromname(name):
-    from modules import sd_models
-    if name == "" or name ==[]: return ""
-    checkpoint_info = sd_models.get_closet_checkpoint_match(name)
-    if checkpoint_info.sha256 is not None:
-        return checkpoint_info.sha256
-    checkpoint_info.calculate_shorthash()
-    return checkpoint_info.sha256
-
-
-################################################
-##### Random
-
-RANCHA = ["R","U","X"]
-
-def randdealer(w:str,randomer,ab,lucks,deep):
-    up,low = lucks["upp"],lucks["low"]
-    up,low = (up.split(","),low.split(","))
-    out = []
-    outd = {"R":[],"U":[],"X":[]}
-    add = RANDMAP[ab]
-    for i, r in enumerate (w.split(",")):
-        if r.strip() =="R":
-            out.append(str(round(randomer[i+add],lucks["round"])))
-        elif r.strip() == "U":
-            out.append(str(round(-2 * randomer[i+add] + 1.5,lucks["round"])))
-        elif r.strip() == "X":
-            out.append(str(round((float(low[i])-float(up[i]))* randomer[i+add] + float(up[i]),lucks["round"])))
-        elif "E" in r:
-            key = r.strip().replace("E","")
-            outd[key].append(BLOCKID[i])
-            out.append("0")
-        else:
-            out.append(r)
-    for key in outd.keys():
-        if outd[key] != []:
-            deep = deep + f",{' '.join(outd[key])}::{key}" if deep else f"{' '.join(outd[key])}::{key}"
-    return ",".join(out), deep
-
-def eratiodealer(dr,randomer,block,num,lucks):
-    if  any(element in dr for element in RANCHA):
-        up,low = lucks["upp"],lucks["low"]
-        up,low = (up.split(","),low.split(","))
-        add = RANDMAP[2]
-        if dr.strip() =="R":
-            return round(randomer[num+add],lucks["round"])
-        elif dr.strip() == "U":
-            return round(-2 * randomer[num+add] + 1,lucks["round"])
-        elif dr.strip() == "X":
-            return round((float(low[block])-float(up[block]))* randomer[num+add] + float(up[block]),lucks["round"])
-    else:
-        return float(dr)
-
-
-################################################
-##### Generate Image
-
-def simggen(s_prompt,s_nprompt,s_steps,s_sampler,s_cfg,s_seed,s_w,s_h,s_batch_size,
-            genoptions,s_hrupscaler,s_hr2ndsteps,s_denois_str,s_hr_scale,
-            mergeinfo,id_sets,modelid,
-            *txt2imgparams,
-            debug = False
-            ):
-    shared.state.begin()
-    from scripts.mergers.components import paramsnames
-    if debug: print(paramsnames)
-
-    #[None, 'Prompt', 'Negative prompt', 'Styles', 'Sampling steps', 'Sampling method', 'Batch count', 'Batch size', 'CFG Scale', 
-    # 'Height', 'Width', 'Hires. fix', 'Denoising strength', 'Upscale by', 'Upscaler', 'Hires steps', 'Resize width to', 'Resize height to', 
-    # 'Hires checkpoint', 'Hires sampling method', 'Hires prompt', 'Hires negative prompt', 'Override settings', 'Script', 'Refiner', 
-    # 'Checkpoint', 'Switch at', 'Seed', 'Extra', 'Variation seed', 'Variation strength', 'Resize seed from width', 'Resize seed from height', '', 'Active', 'Active', 'X Types', 'X Values', 'Y Types', 'Y Values']  
-
-    def g(wanted,wantedv=None):
-        if wanted in paramsnames:return txt2imgparams[paramsnames.index(wanted)]
-        elif wantedv and wantedv in paramsnames:return txt2imgparams[paramsnames.index(wantedv)]
-        else:return None
-
-    sampler_index = g("Sampling method")
-    if type(sampler_index) is str:
-        sampler_name = sampler_index
-    else:       
-        sampler_name = sd_samplers.samplers[sampler_index].name
-
-    hr_sampler_index = g("Hires sampling method")
-    if hr_sampler_index is None: hr_sampler_index = 0
-    if type(sampler_index) is str:
-        hr_sampler_name = hr_sampler_index
-    else:       
-        hr_sampler_name = "Use same sampler" if hr_sampler_index == 0 else  sd_samplers.samplers[hr_sampler_index+1].name
-
-    p = processing.StableDiffusionProcessingTxt2Img(
-        sd_model=shared.sd_model,
-        outpath_samples=opts.outdir_samples or opts.outdir_txt2img_samples,
-        outpath_grids=opts.outdir_grids or opts.outdir_txt2img_grids,
-        prompt=g("Prompt"),
-        styles=g("Styles"),
-        negative_prompt=g('Negative prompt'),
-        seed=g("Seed"),
-        subseed=g("Variation seed"),
-        subseed_strength=g("Variation strength"),
-        seed_resize_from_h=g("Resize seed from height"),
-        seed_resize_from_w=g("Resize seed from width"),
-        seed_enable_extras=g("Extra"),
-        sampler_name=sampler_name,
-        batch_size=g("Batch size"),
-        n_iter=g("Batch count"),
-        steps=g("Sampling steps"),
-        cfg_scale=g("CFG Scale"),
-        width=g("Width"),
-        height=g("Height"),
-        restore_faces=g("Restore faces","Face restore"),
-        tiling=g("Tiling"),
-        enable_hr=g("Hires. fix","Second pass"),
-        hr_scale=g("Upscale by"),
-        hr_upscaler=g("Upscaler"),
-        hr_second_pass_steps=g("Hires steps","Secondary steps"),
-        hr_resize_x=g("Resize width to"),
-        hr_resize_y=g("Resize height to"),
-        override_settings=create_override_settings_dict(g("Override settings")),
-        do_not_save_grid=True,
-        do_not_save_samples=True,
-        do_not_reload_embeddings=True,
-    )
-    p.hr_checkpoint_name=None if g("Hires checkpoint") == 'Use same checkpoint' else g("Hires checkpoint")
-    p.hr_sampler_name=None if hr_sampler_name == 'Use same sampler' else  hr_sampler_name
-
-    if s_sampler is None: s_sampler = 0
-
-    if s_batch_size != 1 :p.batch_size = int(s_batch_size)
-    if s_prompt: p.prompt = s_prompt
-    if s_nprompt: p.negative_prompt = s_nprompt
-    if s_steps: p.steps = s_steps
-    if s_sampler: p.sampler_name = sampler_name
-    if s_cfg: p.cfg_scale = s_cfg
-    if s_seed: p.seed = s_seed
-    if s_w: p.width = s_w
-    if s_h: p.height = s_h
-
-    if not p.cfg_scale: p.cfg_scale = 7
-
-    p.scripts = scripts.scripts_txt2img
-    p.script_args = txt2imgparams[paramsnames.index("Override settings")+1:]
-                
-    p.denoising_strength=g("Denoising strength") if p.enable_hr else None
-
-    p.hr_prompt=g("Hires prompt","Secondary Prompt")
-    p.hr_negative_prompt=g("Hires negative prompt","Secondary negative prompt")
-
-    if "Hires. fix" in genoptions:
-        p.enable_hr = True
-        if s_hrupscaler: p.hr_upscaler = s_hrupscaler
-        if s_hr2ndsteps:p.hr_second_pass_steps = s_hr2ndsteps
-        if s_denois_str:p.denoising_strength = s_denois_str
-        if s_hr_scale:p.hr_scale = s_hr_scale
-
-    if "Restore faces" in genoptions:
-        p.restore_faces = True
-
-    if "Tiling" in genoptions:
-        p.tiling = True
-
-    p.cached_c = [None,None]
-    p.cached_uc = [None,None]
-
-    p.cached_hr_c = [None, None]
-    p.cached_hr_uc = [None, None]
-
-    if type(p.prompt) == list:
-        p.all_prompts = [shared.prompt_styles.apply_styles_to_prompt(x, p.styles) for x in p.prompt]
-    else:
-        p.all_prompts = [shared.prompt_styles.apply_styles_to_prompt(p.prompt, p.styles)]
-
-    if type(p.negative_prompt) == list:
-        p.all_negative_prompts = [shared.prompt_styles.apply_negative_styles_to_prompt(x, p.styles) for x in p.negative_prompt]
-    else:
-        p.all_negative_prompts = [shared.prompt_styles.apply_negative_styles_to_prompt(p.negative_prompt, p.styles)]
-
-    processed:Processed = processing.process_images(p)
-    if "image" in id_sets:
-        for i, image in enumerate(processed.images):
-            processed.images[i] = draw_origin(image, str(modelid),p.width,p.height,p.width)
-
-    if "PNG info" in id_sets:mergeinfo = mergeinfo + " ID " + str(modelid)
-
-    infotext = create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds)
-    if infotext.count("Steps: ")>1:
-        infotext = infotext[:infotext.rindex("Steps")]
-
-    infotexts = infotext.split(",")
-    for i,x in enumerate(infotexts):
-        if "Model:"in x:
-            infotexts[i] = " Model: "+mergeinfo.replace(","," ")
-    infotext= ",".join(infotexts)
-
-    for i, image in enumerate(processed.images):
-        images.save_image(image, opts.outdir_txt2img_samples, "",p.seed, p.prompt,shared.opts.samples_format, p=p,info=infotext)
-
-    if s_batch_size > 1:
-        grid = images.image_grid(processed.images, p.batch_size)
-        processed.images.insert(0, grid)
-        images.save_image(grid, opts.outdir_txt2img_grids, "grid", p.seed, p.prompt, opts.grid_format, info=infotext, short_filename=not opts.grid_extended_filename, p=p, grid=True)
-    shared.state.end()
-    return processed.images,infotext,plaintext_to_html(processed.info), plaintext_to_html(processed.comments),p
-
-
-################################################
-##### Block Ids
-
-def blocker(blocks,blockids):
-    blocks = blocks.split(" ")
-    output = ""
-    for w in blocks:
-        flagger=[False]*len(blockids)
-        changer = True
-        if "-" in w:
-            wt = [wt.strip() for wt in w.split('-')]
-            if  blockids.index(wt[1]) > blockids.index(wt[0]):
-                flagger[blockids.index(wt[0]):blockids.index(wt[1])+1] = [changer]*(blockids.index(wt[1])-blockids.index(wt[0])+1)
-            else:
-                flagger[blockids.index(wt[1]):blockids.index(wt[0])+1] = [changer]*(blockids.index(wt[0])-blockids.index(wt[1])+1)
-        else:
-            output = output + " " + w if output else w
-        for i in range(len(blockids)):
-            if flagger[i]: output = output + " " + blockids[i] if output else blockids[i]
-    return output
-
-
-def blockfromkey(key,isxl):
-    if not isxl:
-        re_inp = re.compile(r'\.input_blocks\.(\d+)\.')  # 12
-        re_mid = re.compile(r'\.middle_block\.(\d+)\.')  # 1
-        re_out = re.compile(r'\.output_blocks\.(\d+)\.') # 12
-
-        weight_index = -1
-
-        NUM_INPUT_BLOCKS = 12
-        NUM_MID_BLOCK = 1
-        NUM_OUTPUT_BLOCKS = 12
-        NUM_TOTAL_BLOCKS = NUM_INPUT_BLOCKS + NUM_MID_BLOCK + NUM_OUTPUT_BLOCKS
-
-        if 'time_embed' in key:
-            weight_index = -2                # before input blocks
-        elif '.out.' in key:
-            weight_index = NUM_TOTAL_BLOCKS - 1     # after output blocks
-        else:
-            m = re_inp.search(key)
-            if m:
-                inp_idx = int(m.groups()[0])
-                weight_index = inp_idx
-            else:
-                m = re_mid.search(key)
-                if m:
-                    weight_index = NUM_INPUT_BLOCKS
-                else:
-                    m = re_out.search(key)
-                    if m:
-                        out_idx = int(m.groups()[0])
-                        weight_index = NUM_INPUT_BLOCKS + NUM_MID_BLOCK + out_idx
-        return BLOCKID[weight_index+1] ,BLOCKID[weight_index+1] 
-
-    else:
-        if not ("weight" in key or "bias" in key):return "Not Merge","Not Merge"
-        if "label_emb" in key or "time_embed" in key: return "Not Merge","Not Merge"
-        if "conditioner.embedders" in key : return "BASE","BASE"
-        if "first_stage_model" in key : return "VAE","BASE"
-        if "model.diffusion_model" in key:
-            if "model.diffusion_model.out." in key: return "OUT8","OUT08"
-            block = re.findall(r'input|mid|output', key)
-            block = block[0].upper().replace("PUT","") if block else ""
-            nums = re.sub(r"\D", "", key)[:1 if "MID" in block else 2] + ("0" if "MID" in block else "")
-            add = re.findall(r"transformer_blocks\.(\d+)\.",key)[0] if "transformer" in key else ""
-            return block + nums + add, block + "0" + nums[0] if "MID" not in block else "M00"
-
-    return "Not Merge", "Not Merge"
-
-################################################
-##### Adjust
-
-def fineman(fine,isxl):
-    if fine.find(",") != -1:
-        tmp = [t.strip() for t in fine.split(",")]
-        fines = [0.0]*8
-        for i,f in enumerate(tmp[0:8]):
-            try:
-                f = float(f)
-                fines[i] = f
-            except Exception:
-                pass
-
-        fine = fines
-    else:
-        return None
-
-    fine = [
-        1 - fine[0] * 0.01,
-        1+ fine[0] * 0.02,
-        1 - fine[1] * 0.01,
-        1+ fine[1] * 0.02,
-        1 - fine[2] * 0.01,
-        [fine[3]*0.02] + colorcalc(fine[4:8],isxl)
-        ]
-    return fine
-
-def colorcalc(cols,isxl):
-    colors = COLSXL if isxl else COLS
-    outs = [[y * cols[i] * 0.02 for y in x] for i,x in enumerate(colors)]
-    return [sum(x) for x in zip(*outs)]
-
-COLS = [[-1,1/3,2/3],[1,1,0],[0,-1,-1],[1,0,1]]
-COLSXL = [[0,0,1],[1,0,0],[-1,-1,0],[-1,1,0]]
-
-def weighttoxl(weight):
-    weight = weight[:9] + weight[12:22] +[0]
-    return weight
-
-FINETUNES = [
-"model.diffusion_model.input_blocks.0.0.weight",
-"model.diffusion_model.input_blocks.0.0.bias",
-"model.diffusion_model.out.0.weight",
-"model.diffusion_model.out.0.bias",
-"model.diffusion_model.out.2.weight",
-"model.diffusion_model.out.2.bias",
-]
-
-################################################
-##### Include/Exclude
-def excluder(block:str,inex:bool,ex_blocks:list,ex_elems:list, key:str):
-    if ex_blocks == [] and ex_elems == [""]:
-        return False
-    out = True if inex == "Include" else False
-    if block in ex_blocks:out = not out
-    if "Adjust" in ex_blocks and key in FINETUNES:out = not out
-    for ke in ex_elems:
-        if ke != "" and ke in key:out = not out
-    if "VAE" in ex_blocks and "first_stage_model"in key:out = not out
-    if "print" in ex_blocks and (out ^ (inex == "Include")):
-        print("Include" if inex else "Exclude",block,ex_blocks,ex_elems,key)
-    return out
-
-################################################
-##### Reset Broken CliP IDs
-
-def resetclip(theta):
-    idkey = "cond_stage_model.transformer.text_model.embeddings.position_ids"
-    broken = []
-    if idkey in theta.keys():
-        correct = torch.Tensor([list(range(77))]).to(torch.int64)
-        current = theta[idkey].to(torch.int64)
-
-        broken = correct.ne(current)
-        broken = [i for i in range(77) if broken[0][i]]
-
-        if broken != []: print("Clip IDs broken and fixed: ",broken)
-        
-        theta[idkey] = correct
-
-
-################################################
-##### cache
-def cachedealer(start):
-    if start:
-        global orig_cache
-        orig_cache = shared.opts.sd_checkpoint_cache
-        shared.opts.sd_checkpoint_cache = 0
-    else:
-        shared.opts.sd_checkpoint_cache = orig_cache
-
-def clearcache():
-    global modelcache
-    del modelcache
-    modelcache = {}
-    gc.collect()
-    devices.torch_gc()
-
-def getcachelist():
-    output = []
-    for key in modelcache.keys():
-        if hasattr(key, "model_name"):
-            output.append(key.model_name)
-    return ",".join(output)
-
-################################################
-##### print
-
-def printstart(model_a,model_b,model_c,base_alpha,base_beta,weights_a,weights_b,mode,useblocks,calcmode,deep,lucks,fine,inex,ex_blocks,ex_elems):
-    print(f"  model A  \t: {model_a}")
-    print(f"  model B  \t: {model_b}")
-    print(f"  model C  \t: {model_c}")
-    print(f"  alpha,beta\t: {base_alpha,base_beta}")
-    print(f"  weights_alpha\t: {weights_a}")
-    print(f"  weights_beta\t: {weights_b}")
-    print(f"  mode\t\t: {mode}")
-    print(f"  MBW \t\t: {useblocks}")
-    print(f"  CalcMode \t: {calcmode}")
-    print(f"  Elemental \t: {deep}")
-    print(f"  Weights Seed\t: {lucks}")
-    print(f"  {inex} \t: {ex_blocks,ex_elems}")
-    print(f"  Adjust \t: {fine}")
-
-def caster(news,hear):
-    if hear: print(news)
-
-def casterr(*args,hear=hear):
-    if hear:
-        names = {id(v): k for k, v in currentframe().f_back.f_locals.items()}
-        print('\n'.join([names.get(id(arg), '???') + ' = ' + repr(arg) for arg in args]))