adding textual inversion adapters

app.py

@@ -161,6 +161,27 @@ def schedulers_change(scheduler, config):
 
     return scheduler_description, config, str(config), assemble_code(config)
     
+def adapters_textual_inversion_change(adapter_textual_inversion, config):
+    
+    if str(adapter_textual_inversion) != 'None' and type(adapter_textual_inversion) != list:
+        
+        adapter_textual_inversion_description = adapters['textual_inversion'][adapter_textual_inversion]['description']
+        in_adapters_textual_inversion_token = adapters['textual_inversion'][adapter_textual_inversion]['token']
+        
+    else:
+        adapter_textual_inversion_description = ""
+        in_adapters_textual_inversion_token = ""
+    
+    config = set_config(config, 'adapter_textual_inversion', adapter_textual_inversion)
+    
+    return adapter_textual_inversion_description, in_adapters_textual_inversion_token, config, str(config), assemble_code(config)
+
+def textual_inversion_token_change(adapter_textual_inversion_token, config):
+    
+    config = set_config(config, 'adapter_textual_inversion_token', adapter_textual_inversion_token)
+
+    return config, str(config), assemble_code(config)
+
 def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True)):
     
     # str_config = str_config.replace("'", '"').replace('None', 'null').replace('False', 'false')
@@ -173,13 +194,25 @@ def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True))
         torch.backends.cuda.matmul.allow_tf32 = get_bool(config["allow_tensorfloat32"]) # Use TensorFloat-32 as of https://huggingface.co/docs/diffusers/main/en/optimization/fp16 faster, but slightly less accurate computations
         
         progress((2,3), desc="Initializing pipeline...")
-                    
+        
+        # INIT PIPELINE
         pipeline = DiffusionPipeline.from_pretrained(
             config["model"], 
             use_safetensors = get_bool(config["use_safetensors"]), 
             torch_dtype = get_data_type(config["data_type"]), 
             variant = get_variant(config["variant"])).to(config["device"])
         
+        if str(config["cpu_offload"]).lower() != 'false':
+            pipeline.enable_model_cpu_offload()
+
+        # AUTO ENCODER
+        if str(config["auto_encoder"]).lower() != 'none': 
+            pipeline.vae = AutoencoderKL.from_pretrained(config["auto_encoder"], torch_dtype=get_data_type(config["data_type"])).to(config["device"])
+
+        if str(config["enable_vae_slicing"]).lower() != 'false': pipeline.enable_vae_slicing()
+        if str(config["enable_vae_tiling"]).lower() != 'false': pipeline.enable_vae_tiling()
+                
+        # INIT REFINER
         if config['refiner'].lower() != 'none':
             refiner = DiffusionPipeline.from_pretrained(
                     config['refiner'],
@@ -189,40 +222,48 @@ def run_inference(config, config_history, progress=gr.Progress(track_tqdm=True))
                     use_safetensors=get_bool(config["use_safetensors"]), 
                     variant = get_variant(config["variant"])).to(config["device"])
             
-        if str(config["safety_checker"]).lower() == 'false':
-            pipeline.safety_checker = None 
+            if str(config["cpu_offload"]).lower() != 'false':
+                refiner.enable_model_cpu_offload()
 
+            if str(config["enable_vae_slicing"]).lower() != 'false': refiner.enable_vae_slicing()
+            if str(config["enable_vae_tiling"]).lower() != 'false': refiner.enable_vae_tiling()
+            
+        # SAFETY CHECKER
+        if str(config["safety_checker"]).lower() == 'false': pipeline.safety_checker = None 
         pipeline.requires_safety_checker = get_bool(config["requires_safety_checker"])
             
+        # SCHEDULER/SOLVER
         pipeline.scheduler = get_scheduler(config["scheduler"], pipeline.scheduler.config)
         
+        # MANUAL SEED/GENERATOR
         if config["manual_seed"] < 0 or config["manual_seed"] is None or config["manual_seed"] == '':
             generator = None
         else:
             generator = torch.manual_seed(int(config["manual_seed"])) 
         
+        # ADAPTERS
+        # TEXTUAL INVERSION
+        if str(config["adapter_textual_inversion"]).lower() != 'none':
+            pipeline.load_textual_inversion(config["adapter_textual_inversion"], token=config["adapter_textual_inversion_token"])
+        
         progress((3,3), desc="Creating the result...")
 
+        prompt = config["prompt"] + config["trigger_token"] + config["adapter_textual_inversion_token"]
+        
         image = pipeline(
-            prompt = config["prompt"],
+            prompt = prompt,
             negative_prompt = config["negative_prompt"],
             generator = generator,
             num_inference_steps = int(config["inference_steps"]), 
             guidance_scale = float(config["guidance_scale"])).images
 
-        if str(config["cpu_offload"]).lower() != 'false':
-            pipeline.enable_model_cpu_offload()
-
         if config['refiner'].lower() != 'none':
             image = refiner(
-                prompt = config["prompt"],
+                prompt = prompt,
                 num_inference_steps = int(config["inference_steps"]), 
                 image=image,
             ).images
             
-            if str(config["cpu_offload"]).lower() != 'false':
-                refiner.enable_model_cpu_offload()
-
         config_history.append(config.copy())
 
         return image[0], dict_list_to_markdown_table(config_history), config_history
@@ -236,6 +277,7 @@ models = appConfig.get("models", {})
 schedulers = appConfig.get("schedulers", {})
 devices =  appConfig.get("devices", [])
 auto_encoders = appConfig.get("auto_encoders", [])
+adapters = appConfig.get("adapters", [])
 
 # interface
 with gr.Blocks(analytics_enabled=False) as demo:
@@ -249,7 +291,7 @@ with gr.Blocks(analytics_enabled=False) as demo:
                 </small>''')
     gr.Markdown("### Device specific settings")
     with gr.Row():
-        in_devices = gr.Dropdown(label="Device:", value=config.value["device"], choices=devices, filterable=True, multiselect=False, allow_custom_value=True)
+        in_devices = gr.Dropdown(label="Device:", value=config.value["device"], choices=devices, filterable=True, multiselect=False, allow_custom_value=True, info="(you may add a custom device address at any time)")
         in_data_type = gr.Radio(label="Data Type:", value=config.value["data_type"], choices=["bfloat16", "float16", "float32"], info="`bfloat16` is not supported on MPS devices right now; `float16` may also not be supported on all devices, Half-precision weights, will save GPU memory, see https://huggingface.co/docs/diffusers/main/en/optimization/fp16")
         in_allow_tensorfloat32 = gr.Radio(label="Allow TensorFloat32:", value=config.value["allow_tensorfloat32"], choices=["True", "False"], info="is not supported on MPS devices right now; use TensorFloat-32 is faster, but results in slightly less accurate computations, see https://huggingface.co/docs/diffusers/main/en/optimization/fp16 ")
         in_variant = gr.Radio(label="Variant:", value=config.value["variant"], choices=["fp16", None], info="Use half-precision weights will save GPU memory, not all models support that, see https://huggingface.co/docs/diffusers/main/en/optimization/fp16 ")
@@ -285,10 +327,10 @@ with gr.Blocks(analytics_enabled=False) as demo:
         in_inference_steps = gr.Number(label="Inference steps", value=config.value["inference_steps"], info="Each step improves the final result but also results in higher computation")
         in_manual_seed = gr.Number(label="Manual seed", value=config.value["manual_seed"], info="Set this to -1 or leave it empty to randomly generate an image. A fixed value will result in a similar image for every run")
         in_guidance_scale = gr.Slider(minimum=0, maximum=100, step=0.1, label="Guidance Scale", value=config.value["guidance_scale"], info="A low guidance scale leads to a faster inference time, with the drawback that negative prompts don’t have any effect on the denoising process.")
+    
+    gr.Markdown("### Auto Encoder")
     with gr.Row():
-        gr.Markdown("### Auto Encoder")
-    with gr.Row():
-        gr.Markdown("**VAE** stands for Variational Autoencoders. An 'autoencoder' is an artificial neural network that is able to encode input data and decode to output data to bascially recreate the input. The VAE whereas adds a couple of additional layers of complexity to create new and unique output.")
+        gr.Markdown("**VAE** stands for Variational Auto Encoders. An 'autoencoder' is an artificial neural network that is able to encode input data and decode to output data to bascially recreate the input. The VAE whereas adds a couple of additional layers of complexity to create new and unique output.")
     with gr.Row():
         with gr.Column():
             in_auto_encoders = gr.Dropdown(value="None", choices=list(auto_encoders.keys()), label="Auto encoder", info="leave empty to not add an auto encoder")
@@ -297,6 +339,18 @@ with gr.Blocks(analytics_enabled=False) as demo:
             in_enable_vae_slicing = gr.Radio(label="Enable VAE slicing:", value=config.value["enable_vae_slicing"], choices=["True", "False"], info="decoding the batches of latents one image at a time, which may reduce memory usage, see https://huggingface.co/docs/diffusers/main/en/optimization/memory")
             in_enable_vae_tiling= gr.Radio(label="Enable VAE tiling:", value=config.value["enable_vae_tiling"], choices=["True", "False"], info="splitting the image into overlapping tiles, decoding the tiles, and then blending the outputs together to compose the final image, see https://huggingface.co/docs/diffusers/main/en/optimization/memory")
         
+    gr.Markdown("### Adapters")
+    with gr.Row():
+        gr.Markdown('''Adapters allow you to apply finetuned weights to your base model. They come in many flavors depending on how they were trained. See see https://huggingface.co/docs/diffusers/using-diffusers/loading_adapters''')
+    with gr.Row():
+        gr.Markdown('#### Textual Inversion Adapters')
+    with gr.Row():
+        gr.Markdown('(a technique that enables a model like Stable Diffusion to learn a new concept from just a few sample images)')
+    with gr.Row():
+        in_adapters_textual_inversion = gr.Dropdown(value="None", choices=list(adapters['textual_inversion'].keys()), label="Adapter", info="leave empty to not use an adapter")
+        in_adapters_textual_inversion_token = gr.Textbox(value="None", label="Adapter token", info="required to activate the token, will be added to your prompt")
+        out_adapters_textual_inversion_description = gr.Textbox(value="", label="Description")
+        
     gr.Markdown("### Output")
     with gr.Row():
         btn_start_pipeline = gr.Button(value="Run", variant="primary")
@@ -325,6 +379,8 @@ with gr.Blocks(analytics_enabled=False) as demo:
     in_auto_encoders.change(auto_encoders_change, inputs=[in_auto_encoders, config], outputs=[out_auto_encoder_description, config, out_config, out_code])
     in_enable_vae_slicing.change(enable_vae_slicing_change, inputs=[in_enable_vae_slicing, config], outputs=[config, out_config, out_code])
     in_enable_vae_tiling.change(enable_vae_tiling_change, inputs=[in_enable_vae_tiling, config], outputs=[config, out_config, out_code])
+    in_adapters_textual_inversion.change(adapters_textual_inversion_change, inputs=[in_adapters_textual_inversion, config], outputs=[out_adapters_textual_inversion_description, in_adapters_textual_inversion_token, config, out_config, out_code])
+    in_adapters_textual_inversion_token.change(textual_inversion_token_change, inputs=[in_adapters_textual_inversion_token, config], outputs=[config, out_config, out_code])
     in_prompt.change(prompt_change, inputs=[in_prompt, config], outputs=[config, out_config, out_code])
     in_trigger_token.change(trigger_token_change, inputs=[in_trigger_token, config], outputs=[config, out_config, out_code])
     in_negative_prompt.change(negative_prompt_change, inputs=[in_negative_prompt, config], outputs=[config, out_config, out_code])
@@ -354,7 +410,8 @@ with gr.Blocks(analytics_enabled=False) as demo:
             in_negative_prompt,
             in_inference_steps,
             in_manual_seed,
-            in_guidance_scale
+            in_guidance_scale,
+            in_adapters_textual_inversion
             ])
 
 demo.launch(show_error=True)

appConfig.json

@@ -78,6 +78,15 @@
         "madebyollin/sdxl-vae-fp16-fix": "stable diffusion models encoder with fp16 precision, see https://huggingface.co/madebyollin/sdxl-vae-fp16-fix",
         "stabilityai/sd-vae-ft-mse": "works best with CompVis/stable-diffusion-v1-4, see https://huggingface.co/stabilityai/sd-vae-ft-mse"
     },
+    "adapters": {
+        "textual_inversion": {
+            "None": {"token": "", "description": ""},
+            "sd-concepts-library/gta5-artwork": {
+                "token": "<gta-artwork>",
+                "description": "see https://huggingface.co/sd-concepts-library/gta5-artwork"
+            }
+        }
+    },
     "schedulers": {
         "DDPMScheduler":                    "Denoising Diffusion Probabilistic Model",  
         "DDIMScheduler":                    "Denoising Diffusion Incremental Sampling, efficient image generation, might require more tunin",  

config.py

@@ -52,6 +52,8 @@ def get_initial_config():
         "manual_seed": 42,
         "inference_steps": 10,
         "guidance_scale": 5,
+        "adapter_textual_inversion": None,
+        "adapter_textual_inversion_token": None,
         "prompt": 'A white rabbit',
         "trigger_token": '',
         "negative_prompt": 'lowres, cropped, worst quality, low quality',
@@ -98,7 +100,9 @@ def get_config_from_url(initial_config, request: Request):
             return_config['negative_prompt'],
             return_config['inference_steps'],
             return_config['manual_seed'],
-            return_config['guidance_scale']
+            return_config['guidance_scale'],
+            return_config['adapter_textual_inversion'],
+            return_config['adapter_textual_inversion_token']
             ]
         
 def load_app_config():
@@ -137,6 +141,7 @@ def assemble_code(str_config):
         code.append('data_type = torch.bfloat16')
     else:
         code.append('data_type = torch.float16')
+        
     code.append(f'torch.backends.cuda.matmul.allow_tf32 = {config["allow_tensorfloat32"]}')
     
     if str(config["variant"]) == 'None':
@@ -144,22 +149,25 @@ def assemble_code(str_config):
     else:
         code.append(f'variant = "{config["variant"]}"')
     
-        
     code.append(f'''use_safetensors = {config["use_safetensors"]}''')
         
+    # INIT PIPELINE
     code.append(f'''pipeline = DiffusionPipeline.from_pretrained(
             "{config['model']}", 
             use_safetensors=use_safetensors, 
             torch_dtype=data_type, 
             variant=variant).to(device)''')
     
+    if str(config["cpu_offload"]).lower() != 'false': code.append("pipeline.enable_model_cpu_offload()")
+    
+    # AUTO ENCODER
     if str(config["auto_encoder"]).lower() != 'none': 
         code.append(f'pipeline.vae = AutoencoderKL.from_pretrained("{config["auto_encoder"]}", torch_dtype=data_type).to(device)')
     
-    if str(config["cpu_offload"]).lower() != 'false': code.append("pipeline.enable_model_cpu_offload()")
     if str(config["enable_vae_slicing"]).lower() != 'false': code.append("pipeline.enable_vae_slicing()")
     if str(config["enable_vae_tiling"]).lower() != 'false': code.append("pipeline.enable_vae_tiling()")
     
+    # INIT REFINER
     if config['refiner'].lower() != 'none':
         code.append(f'''refiner = DiffusionPipeline.from_pretrained(
                 "{config['refiner']}",
@@ -174,13 +182,16 @@ def assemble_code(str_config):
         if str(config["enable_vae_slicing"]).lower() != 'false': code.append("refiner.enable_vae_slicing()")
         if str(config["enable_vae_tiling"]).lower() != 'false': code.append("refiner.enable_vae_tiling()")
         
+    # SAFETY CHECKER
     code.append(f'pipeline.requires_safety_checker = {config["requires_safety_checker"]}')
-
     if str(config["safety_checker"]).lower() == 'false':
         code.append(f'pipeline.safety_checker = None')
     
-    code.append(f'pipeline.scheduler = {config["scheduler"]}.from_config(pipeline.scheduler.config)')
+    # SCHEDULER/SOLVER
+    if str(config["scheduler"]).lower() != 'none':
+        code.append(f'pipeline.scheduler = {config["scheduler"]}.from_config(pipeline.scheduler.config)')
     
+    # MANUAL SEED/GENERATOR
     if config['manual_seed'] < 0 or config['manual_seed'] is None or config['manual_seed'] == '':
         code.append(f'# manual_seed = {config["manual_seed"]}')
         code.append(f'generator = None')
@@ -188,7 +199,10 @@ def assemble_code(str_config):
         code.append(f'manual_seed = {config["manual_seed"]}')
         code.append(f'generator = torch.manual_seed(manual_seed)')
 
-    code.append(f'prompt = "{config["prompt"]} {config["trigger_token"]}"')
+    if str(config["adapter_textual_inversion"]).lower() != 'none':
+        code.append(f'pipeline.load_textual_inversion("{config["adapter_textual_inversion"]}", token="{config["adapter_textual_inversion_token"]}")')
+
+    code.append(f'prompt = "{config["prompt"]} {config["trigger_token"]} {config["adapter_textual_inversion_token"]}"')
     code.append(f'negative_prompt = "{config["negative_prompt"]}"')
     code.append(f'inference_steps = {config["inference_steps"]}')
     code.append(f'guidance_scale = {config["guidance_scale"]}')