# Copyright 2024 NVIDIA CORPORATION & AFFILIATES # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 import os import random import re import numpy as np import torch from diffusion.utils.logger import get_root_logger from tools.download import find_model def save_checkpoint( work_dir, epoch, model, model_ema=None, optimizer=None, lr_scheduler=None, generator=torch.Generator(device="cpu").manual_seed(42), keep_last=False, step=None, add_symlink=False, ): os.makedirs(work_dir, exist_ok=True) state_dict = dict(state_dict=model.state_dict()) if model_ema is not None: state_dict["state_dict_ema"] = model_ema.state_dict() if optimizer is not None: state_dict["optimizer"] = optimizer.state_dict() if lr_scheduler is not None: state_dict["scheduler"] = lr_scheduler.state_dict() if epoch is not None: state_dict["epoch"] = epoch file_path = os.path.join(work_dir, f"epoch_{epoch}.pth") if step is not None: file_path = file_path.split(".pth")[0] + f"_step_{step}.pth" rng_state = { "torch": torch.get_rng_state(), "torch_cuda": torch.cuda.get_rng_state_all(), "numpy": np.random.get_state(), "python": random.getstate(), "generator": generator.get_state(), } state_dict["rng_state"] = rng_state logger = get_root_logger() torch.save(state_dict, file_path) logger.info(f"Saved checkpoint of epoch {epoch} to {file_path.format(epoch)}.") if keep_last: for i in range(epoch): previous_ckgt = file_path.format(i) if os.path.exists(previous_ckgt): os.remove(previous_ckgt) if add_symlink: link_path = os.path.join(os.path.dirname(file_path), "latest.pth") if os.path.exists(link_path) or os.path.islink(link_path): os.remove(link_path) os.symlink(os.path.abspath(file_path), link_path) return file_path def load_checkpoint( checkpoint, model, model_ema=None, optimizer=None, lr_scheduler=None, load_ema=False, resume_optimizer=True, resume_lr_scheduler=True, null_embed_path=None, ): assert isinstance(checkpoint, str) logger = get_root_logger() ckpt_file = checkpoint checkpoint = find_model(ckpt_file) state_dict_keys = ["pos_embed", "base_model.pos_embed", "model.pos_embed"] for key in state_dict_keys: if key in checkpoint["state_dict"]: del checkpoint["state_dict"][key] if "state_dict_ema" in checkpoint and key in checkpoint["state_dict_ema"]: del checkpoint["state_dict_ema"][key] break if load_ema: state_dict = checkpoint["state_dict_ema"] else: state_dict = checkpoint.get("state_dict", checkpoint) # to be compatible with the official checkpoint null_embed = torch.load(null_embed_path, map_location="cpu") state_dict["y_embedder.y_embedding"] = null_embed["uncond_prompt_embeds"][0] rng_state = checkpoint.get("rng_state", None) missing, unexpect = model.load_state_dict(state_dict, strict=False) if model_ema is not None: model_ema.load_state_dict(checkpoint["state_dict_ema"], strict=False) if optimizer is not None and resume_optimizer: optimizer.load_state_dict(checkpoint["optimizer"]) if lr_scheduler is not None and resume_lr_scheduler: lr_scheduler.load_state_dict(checkpoint["scheduler"]) epoch = 0 if optimizer is not None: epoch = checkpoint.get("epoch", re.match(r".*epoch_(\d*).*.pth", ckpt_file).group()[0]) logger.info( f"Resume checkpoint of epoch {epoch} from {ckpt_file}. Load ema: {load_ema}, " f"resume optimizer: {resume_optimizer}, resume lr scheduler: {resume_lr_scheduler}." ) return epoch, missing, unexpect, rng_state logger.info(f"Load checkpoint from {ckpt_file}. Load ema: {load_ema}.") return epoch, missing, unexpect, rng_state