release ready
Browse files- app.py +8 -57
- ditail/src/ditail_demo.py +2 -2
app.py
CHANGED
@@ -13,7 +13,6 @@ from ditail import DitailDemo, seed_everything
|
|
13 |
|
14 |
BASE_MODEL = {
|
15 |
'sd1.5': 'runwayml/stable-diffusion-v1-5',
|
16 |
-
# 'sd1.5': './ditail/model/stable-diffusion-v1-5'
|
17 |
'realistic vision': 'stablediffusionapi/realistic-vision-v51',
|
18 |
'pastel mix (anime)': 'stablediffusionapi/pastel-mix-stylized-anime',
|
19 |
# 'chaos (abstract)': 'MAPS-research/Chaos3.0',
|
@@ -27,7 +26,6 @@ LORA_TRIGGER_WORD = {
|
|
27 |
'flat': ['sdh', 'flat illustration'],
|
28 |
'minecraft': ['minecraft square style', 'cg, computer graphics'],
|
29 |
'animeoutline': ['lineart', 'monochrome'],
|
30 |
-
# 'caravaggio': ['oil painting', 'in the style of caravaggio'],
|
31 |
'impressionism': ['impressionist', 'in the style of Monet'],
|
32 |
'pop': ['POP ART'],
|
33 |
'shinkai_makoto': ['shinkai makoto', 'kimi no na wa.', 'tenki no ko', 'kotonoha no niwa'],
|
@@ -62,6 +60,7 @@ class WebApp():
|
|
62 |
self.args_input = {} # for gr.components only
|
63 |
self.gr_loras = list(LORA_TRIGGER_WORD.keys())
|
64 |
|
|
|
65 |
self.gtag = os.environ.get('GTag')
|
66 |
|
67 |
self.ga_script = f"""
|
@@ -83,7 +82,6 @@ class WebApp():
|
|
83 |
self.debug_mode = debug_mode # turn off clip interrogator when debugging for faster building speed
|
84 |
if not self.debug_mode:
|
85 |
self.init_interrogator()
|
86 |
-
|
87 |
|
88 |
|
89 |
def init_interrogator(self):
|
@@ -135,18 +133,15 @@ class WebApp():
|
|
135 |
self.args_input['img'] = gr.Image(label='content image', type='pil', show_share_button=False, elem_classes="input_image")
|
136 |
|
137 |
def get_prompts(self):
|
138 |
-
# with gr.Row():
|
139 |
generate_prompt = gr.Checkbox(label='generate prompt with clip', value=True)
|
140 |
self.args_input['pos_prompt'] = gr.Textbox(label='prompt')
|
141 |
-
|
142 |
-
|
143 |
# event listeners
|
144 |
self.args_input['img'].upload(self._interrogate_image, inputs=[self.args_input['img'], generate_prompt], outputs=[self.args_input['pos_prompt']])
|
145 |
generate_prompt.change(self._interrogate_image, inputs=[self.args_input['img'], generate_prompt], outputs=[self.args_input['pos_prompt']])
|
146 |
|
147 |
|
148 |
def _interrogate_image(self, image, generate_prompt):
|
149 |
-
# self.init_interrogator()
|
150 |
if hasattr(self, 'ci') and generate_prompt:
|
151 |
return self.ci.interrogate_fast(image).split(',')[0].replace('arafed', '')
|
152 |
else:
|
@@ -169,7 +164,6 @@ class WebApp():
|
|
169 |
with gr.Column():
|
170 |
self.args_input['inv_model'] = gr.Radio(choices=list(BASE_MODEL.keys()), value=list(BASE_MODEL.keys())[1], label='inversion base model')
|
171 |
self.args_input['neg_prompt'] = gr.Textbox(label='negative prompt', value=self.args_base['neg_prompt'])
|
172 |
-
# with gr.Row():
|
173 |
self.args_input['alpha'] = gr.Number(label='positive prompt scaling weight (alpha)', value=self.args_base['alpha'], interactive=True)
|
174 |
self.args_input['beta'] = gr.Number(label='negative prompt scaling weight (beta)', value=self.args_base['beta'], interactive=True)
|
175 |
|
@@ -185,40 +179,6 @@ class WebApp():
|
|
185 |
self.args_input['seed'] = gr.Number(label='seed', value=self.args_base['seed'], interactive=True, precision=0, step=1)
|
186 |
|
187 |
def run_ditail(self, *values):
|
188 |
-
try:
|
189 |
-
self.args = self.args_base.copy()
|
190 |
-
print(self.args_input.keys())
|
191 |
-
for k, v in zip(list(self.args_input.keys()), values):
|
192 |
-
self.args[k] = v
|
193 |
-
# quick fix for example
|
194 |
-
self.args['lora'] = 'none' if not isinstance(self.args['lora'], str) else self.args['lora']
|
195 |
-
print('selected lora: ', self.args['lora'])
|
196 |
-
# map inversion model to url
|
197 |
-
self.args['pos_prompt'] = ', '.join(LORA_TRIGGER_WORD.get(self.args['lora'], [])+[self.args['pos_prompt']])
|
198 |
-
self.args['inv_model'] = BASE_MODEL[self.args['inv_model']]
|
199 |
-
self.args['spl_model'] = BASE_MODEL[self.args['spl_model']]
|
200 |
-
print('selected model: ', self.args['inv_model'], self.args['spl_model'])
|
201 |
-
|
202 |
-
seed_everything(self.args['seed'])
|
203 |
-
ditail = DitailDemo(self.args)
|
204 |
-
|
205 |
-
metadata_to_show = ['inv_model', 'spl_model', 'lora', 'lora_scale', 'inv_steps', 'spl_steps', 'pos_prompt', 'alpha', 'neg_prompt', 'beta', 'omega']
|
206 |
-
self.args_to_show = {}
|
207 |
-
for key in metadata_to_show:
|
208 |
-
self.args_to_show[key] = self.args[key]
|
209 |
-
|
210 |
-
img = ditail.run_ditail()
|
211 |
-
|
212 |
-
# reset ditail
|
213 |
-
ditail = None
|
214 |
-
|
215 |
-
return img, self.args_to_show
|
216 |
-
# return self.args['img'], self.args
|
217 |
-
except Exception as e:
|
218 |
-
print(f"Error catched: {e}")
|
219 |
-
gr.Markdown(f"**Error catched: {e}**")
|
220 |
-
|
221 |
-
def run_ditail_alt(self, *values):
|
222 |
gr_args = self.args_base.copy()
|
223 |
print(self.args_input.keys())
|
224 |
for k, v in zip(list(self.args_input.keys()), values):
|
@@ -242,26 +202,19 @@ class WebApp():
|
|
242 |
|
243 |
img = ditail.run_ditail()
|
244 |
|
245 |
-
# reset ditail
|
246 |
ditail = None
|
247 |
|
248 |
return img, args_to_show
|
249 |
|
250 |
def run_example(self, img, prompt, inv_model, spl_model, lora):
|
251 |
-
return self.
|
252 |
|
253 |
def show_credits(self):
|
254 |
-
# gr.Markdown(
|
255 |
-
# """
|
256 |
-
# ### About Diffusion Cocktail (Ditail)
|
257 |
-
# * This is a research project by [MAPS Lab](https://whongyi.github.io/MAPS-research), [NYU Shanghai](https://shanghai.nyu.edu)
|
258 |
-
# * Authors: Haoming Liu ([email protected]), Yuanhe Guo ([email protected]), Hongyi Wen ([email protected])
|
259 |
-
# """
|
260 |
-
# )
|
261 |
gr.Markdown(
|
262 |
"""
|
263 |
### Model Credits
|
264 |
-
* Diffusion Models are downloaded from [huggingface](https://huggingface.co)
|
265 |
* LoRA Models are downloaded from [civitai](https://civitai.com) and [liblib](https://www.liblib.art): [film](https://civitai.com/models/90393/japan-vibes-film-color), [snow](https://www.liblib.art/modelinfo/f732b23b02f041bdb7f8f3f8a256ca8b), [flat](https://www.liblib.art/modelinfo/76dcb8b59d814960b0244849f2747a15), [minecraft](https://civitai.com/models/113741/minecraft-square-style), [animeoutline](https://civitai.com/models/16014/anime-lineart-manga-like-style), [impressionism](https://civitai.com/models/113383/y5-impressionism-style), [pop](https://civitai.com/models/161450?modelVersionId=188417), [shinkai_makoto](https://civitai.com/models/10626?modelVersionId=12610)
|
266 |
"""
|
267 |
)
|
@@ -272,7 +225,6 @@ class WebApp():
|
|
272 |
|
273 |
self.title()
|
274 |
with gr.Row():
|
275 |
-
# with gr.Column():
|
276 |
self.get_image()
|
277 |
|
278 |
with gr.Column():
|
@@ -286,10 +238,10 @@ class WebApp():
|
|
286 |
|
287 |
with gr.Row():
|
288 |
output_image = gr.Image(label="output image")
|
289 |
-
|
290 |
-
|
291 |
|
292 |
-
submit_btn.click(self.
|
293 |
inputs=list(self.args_input.values()),
|
294 |
outputs=[output_image, metadata],
|
295 |
scroll_to_output=True,
|
@@ -318,6 +270,5 @@ demo = app.ui()
|
|
318 |
|
319 |
if __name__ == "__main__":
|
320 |
demo.launch(share=True)
|
321 |
-
# demo.launch()
|
322 |
|
323 |
|
|
|
13 |
|
14 |
BASE_MODEL = {
|
15 |
'sd1.5': 'runwayml/stable-diffusion-v1-5',
|
|
|
16 |
'realistic vision': 'stablediffusionapi/realistic-vision-v51',
|
17 |
'pastel mix (anime)': 'stablediffusionapi/pastel-mix-stylized-anime',
|
18 |
# 'chaos (abstract)': 'MAPS-research/Chaos3.0',
|
|
|
26 |
'flat': ['sdh', 'flat illustration'],
|
27 |
'minecraft': ['minecraft square style', 'cg, computer graphics'],
|
28 |
'animeoutline': ['lineart', 'monochrome'],
|
|
|
29 |
'impressionism': ['impressionist', 'in the style of Monet'],
|
30 |
'pop': ['POP ART'],
|
31 |
'shinkai_makoto': ['shinkai makoto', 'kimi no na wa.', 'tenki no ko', 'kotonoha no niwa'],
|
|
|
60 |
self.args_input = {} # for gr.components only
|
61 |
self.gr_loras = list(LORA_TRIGGER_WORD.keys())
|
62 |
|
63 |
+
# fun fact: google analytics doesn't work in this space currently
|
64 |
self.gtag = os.environ.get('GTag')
|
65 |
|
66 |
self.ga_script = f"""
|
|
|
82 |
self.debug_mode = debug_mode # turn off clip interrogator when debugging for faster building speed
|
83 |
if not self.debug_mode:
|
84 |
self.init_interrogator()
|
|
|
85 |
|
86 |
|
87 |
def init_interrogator(self):
|
|
|
133 |
self.args_input['img'] = gr.Image(label='content image', type='pil', show_share_button=False, elem_classes="input_image")
|
134 |
|
135 |
def get_prompts(self):
|
|
|
136 |
generate_prompt = gr.Checkbox(label='generate prompt with clip', value=True)
|
137 |
self.args_input['pos_prompt'] = gr.Textbox(label='prompt')
|
138 |
+
|
|
|
139 |
# event listeners
|
140 |
self.args_input['img'].upload(self._interrogate_image, inputs=[self.args_input['img'], generate_prompt], outputs=[self.args_input['pos_prompt']])
|
141 |
generate_prompt.change(self._interrogate_image, inputs=[self.args_input['img'], generate_prompt], outputs=[self.args_input['pos_prompt']])
|
142 |
|
143 |
|
144 |
def _interrogate_image(self, image, generate_prompt):
|
|
|
145 |
if hasattr(self, 'ci') and generate_prompt:
|
146 |
return self.ci.interrogate_fast(image).split(',')[0].replace('arafed', '')
|
147 |
else:
|
|
|
164 |
with gr.Column():
|
165 |
self.args_input['inv_model'] = gr.Radio(choices=list(BASE_MODEL.keys()), value=list(BASE_MODEL.keys())[1], label='inversion base model')
|
166 |
self.args_input['neg_prompt'] = gr.Textbox(label='negative prompt', value=self.args_base['neg_prompt'])
|
|
|
167 |
self.args_input['alpha'] = gr.Number(label='positive prompt scaling weight (alpha)', value=self.args_base['alpha'], interactive=True)
|
168 |
self.args_input['beta'] = gr.Number(label='negative prompt scaling weight (beta)', value=self.args_base['beta'], interactive=True)
|
169 |
|
|
|
179 |
self.args_input['seed'] = gr.Number(label='seed', value=self.args_base['seed'], interactive=True, precision=0, step=1)
|
180 |
|
181 |
def run_ditail(self, *values):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
182 |
gr_args = self.args_base.copy()
|
183 |
print(self.args_input.keys())
|
184 |
for k, v in zip(list(self.args_input.keys()), values):
|
|
|
202 |
|
203 |
img = ditail.run_ditail()
|
204 |
|
205 |
+
# reset ditail to free memory usage
|
206 |
ditail = None
|
207 |
|
208 |
return img, args_to_show
|
209 |
|
210 |
def run_example(self, img, prompt, inv_model, spl_model, lora):
|
211 |
+
return self.run_ditail(img, prompt, spl_model, gr.State(lora), inv_model)
|
212 |
|
213 |
def show_credits(self):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
214 |
gr.Markdown(
|
215 |
"""
|
216 |
### Model Credits
|
217 |
+
* Diffusion Models are downloaded from [huggingface](https://huggingface.co): [stable diffusion 1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5), [realistic vision](https://huggingface.co/stablediffusionapi/realistic-vision-v51), [pastel mix](https://huggingface.co/stablediffusionapi/pastel-mix-stylized-anime)
|
218 |
* LoRA Models are downloaded from [civitai](https://civitai.com) and [liblib](https://www.liblib.art): [film](https://civitai.com/models/90393/japan-vibes-film-color), [snow](https://www.liblib.art/modelinfo/f732b23b02f041bdb7f8f3f8a256ca8b), [flat](https://www.liblib.art/modelinfo/76dcb8b59d814960b0244849f2747a15), [minecraft](https://civitai.com/models/113741/minecraft-square-style), [animeoutline](https://civitai.com/models/16014/anime-lineart-manga-like-style), [impressionism](https://civitai.com/models/113383/y5-impressionism-style), [pop](https://civitai.com/models/161450?modelVersionId=188417), [shinkai_makoto](https://civitai.com/models/10626?modelVersionId=12610)
|
219 |
"""
|
220 |
)
|
|
|
225 |
|
226 |
self.title()
|
227 |
with gr.Row():
|
|
|
228 |
self.get_image()
|
229 |
|
230 |
with gr.Column():
|
|
|
238 |
|
239 |
with gr.Row():
|
240 |
output_image = gr.Image(label="output image")
|
241 |
+
with gr.Column():
|
242 |
+
metadata = gr.JSON(label='metadata')
|
243 |
|
244 |
+
submit_btn.click(self.run_ditail,
|
245 |
inputs=list(self.args_input.values()),
|
246 |
outputs=[output_image, metadata],
|
247 |
scroll_to_output=True,
|
|
|
270 |
|
271 |
if __name__ == "__main__":
|
272 |
demo.launch(share=True)
|
|
|
273 |
|
274 |
|
ditail/src/ditail_demo.py
CHANGED
@@ -57,8 +57,8 @@ class DitailDemo(nn.Module):
|
|
57 |
self.spl_model, torch_dtype=torch.float16
|
58 |
).to(self.device)
|
59 |
if self.lora != 'none':
|
60 |
-
pipe.unfuse_lora()
|
61 |
-
pipe.unload_lora_weights()
|
62 |
pipe.load_lora_weights(self.lora_dir, weight_name=f'{self.lora}.safetensors')
|
63 |
pipe.fuse_lora(lora_scale=self.lora_scale)
|
64 |
pipe.enable_xformers_memory_efficient_attention()
|
|
|
57 |
self.spl_model, torch_dtype=torch.float16
|
58 |
).to(self.device)
|
59 |
if self.lora != 'none':
|
60 |
+
# pipe.unfuse_lora()
|
61 |
+
# pipe.unload_lora_weights()
|
62 |
pipe.load_lora_weights(self.lora_dir, weight_name=f'{self.lora}.safetensors')
|
63 |
pipe.fuse_lora(lora_scale=self.lora_scale)
|
64 |
pipe.enable_xformers_memory_efficient_attention()
|