Scheduled Commit

ai-toolkit.log

@@ -1,3 +1,176 @@
 The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.
 Running 1 job
 
+/usr/local/lib/python3.10/dist-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 1.4.23 (you have 1.4.15). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1.
+  check_for_updates()
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/mediapipe_face/mediapipe_face_common.py:7: UserWarning: The module 'mediapipe' is not installed. The package will have limited functionality. Please install it using the command: pip install 'mediapipe'
+  warnings.warn(
+/usr/local/lib/python3.10/dist-packages/timm/models/layers/__init__.py:48: FutureWarning: Importing from timm.models.layers is deprecated, please import via timm.layers
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.layers", FutureWarning)
+/usr/local/lib/python3.10/dist-packages/timm/models/registry.py:4: FutureWarning: Importing from timm.models.registry is deprecated, please import via timm.models
+  warnings.warn(f"Importing from {__name__} is deprecated, please import via timm.models", FutureWarning)
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_5m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_5m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
+  return register_model(fn_wrapper)
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_11m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_11m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
+  return register_model(fn_wrapper)
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_224 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_224. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
+  return register_model(fn_wrapper)
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_384 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_384. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
+  return register_model(fn_wrapper)
+/usr/local/lib/python3.10/dist-packages/controlnet_aux/segment_anything/modeling/tiny_vit_sam.py:654: UserWarning: Overwriting tiny_vit_21m_512 in registry with controlnet_aux.segment_anything.modeling.tiny_vit_sam.tiny_vit_21m_512. This is because the name being registered conflicts with an existing name. Please check if this is not expected.
+  return register_model(fn_wrapper)
+/workspace/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py:61: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
+  self.scaler = torch.cuda.amp.GradScaler()
+You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers
+{
+    "type": "sd_trainer",
+    "training_folder": "output",
+    "device": "cuda:0",
+    "network": {
+        "type": "lora",
+        "linear": 16,
+        "linear_alpha": 16
+    },
+    "save": {
+        "dtype": "float16",
+        "save_every": 500,
+        "max_step_saves_to_keep": 4,
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/images",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "shuffle_tokens": false,
+            "cache_latents_to_disk": true,
+            "resolution": [
+                512,
+                768,
+                1024
+            ]
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "steps": 2000,
+        "gradient_accumulation_steps": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "lr": 0.0004,
+        "ema_config": {
+            "use_ema": true,
+            "ema_decay": 0.99
+        },
+        "dtype": "bf16"
+    },
+    "model": {
+        "name_or_path": "black-forest-labs/FLUX.1-dev",
+        "is_flux": true,
+        "quantize": true
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 500,
+        "width": 1024,
+        "height": 1024,
+        "prompts": [
+            "Photo of xtina holding a sign that says 'I LOVE PROMPTS!'",
+            "Professional headshot of xtina in a business suit.",
+            "A happy pilot xtina of a Boeing 747.",
+            "A doctor xtina talking to a patient.",
+            "A chef xtina in the middle of a bustling kitchen, plating a beautifully arranged dish.",
+            "A young xtina with a big grin, holding a large ice cream cone in front of an old-fashioned ice cream parlor.",
+            "A person xtina in a tuxedo, looking directly into the camera with a confident smile, standing on a red carpet at a gala event.",
+            "Person xtina with a bitchin' 80's mullet hairstyle leaning out the window of a pontiac firebird"
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 4,
+        "sample_steps": 20
+    },
+    "trigger_word": "xtina"
+}
+Using EMA
+
+#############################################
+# Running job: my_first_flux_lora_v1
+#############################################
+
+
+Running  1 process
+Loading Flux model
+Loading transformer
+Quantizing transformer
+Loading vae
+Loading t5
+
+
+Quantizing T5
+Loading clip
+making pipe
+preparing
+create LoRA network. base dim (rank): 16, alpha: 16
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 494 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/images
+  -  Preprocessing image dimensions
+
  0%|          | 0/40 [00:00<?, ?it/s]
 10%|█         | 4/40 [00:00<00:00, 37.15it/s]
 20%|██        | 8/40 [00:00<00:01, 24.03it/s]
 32%|███▎      | 13/40 [00:00<00:00, 28.82it/s]
 57%|█████▊    | 23/40 [00:00<00:00, 49.26it/s]
 75%|███████▌  | 30/40 [00:00<00:00, 53.40it/s]
 90%|█████████ | 36/40 [00:00<00:00, 46.91it/s]
+  -  Found 40 images
+Bucket sizes for /workspace/ai-toolkit/images:
+384x576: 18 files
+448x512: 1 files
+448x576: 13 files
+576x448: 5 files
+384x640: 1 files
+512x512: 2 files
+6 buckets made
+Caching latents for /workspace/ai-toolkit/images
+ - Saving latents to disk
+
+Dataset: /workspace/ai-toolkit/images
+  -  Preprocessing image dimensions
+
  0%|          | 0/40 [00:00<?, ?it/s]
+  -  Found 40 images
+Bucket sizes for /workspace/ai-toolkit/images:
+576x832: 12 files
+640x768: 6 files
+640x832: 8 files
+576x896: 7 files
+832x640: 3 files
+768x640: 2 files
+704x768: 1 files
+768x768: 1 files
+8 buckets made
+Caching latents for /workspace/ai-toolkit/images
+ - Saving latents to disk
+
+Dataset: /workspace/ai-toolkit/images
+  -  Preprocessing image dimensions
+
  0%|          | 0/40 [00:00<?, ?it/s]
+  -  Found 40 images
+Bucket sizes for /workspace/ai-toolkit/images:
+832x1216: 12 files
+896x1088: 6 files
+896x1152: 5 files
+832x1152: 6 files
+768x1280: 2 files
+1152x832: 2 files
+768x1152: 1 files
+704x1024: 1 files
+1088x896: 2 files
+1152x896: 1 files
+960x1024: 1 files
+1024x1024: 1 files
+12 buckets made
+Caching latents for /workspace/ai-toolkit/images
+ - Saving latents to disk
+
+Generating baseline samples before training
+