+ 
+
+
+
+### Finetune Llama 3.2, Mistral, Phi-3.5, Qwen 2.5 & Gemma 2-5x faster with 80% less memory!
+
+
+
+
+
+
+
+
+### Finetune Llama 3.2, Mistral, Phi-3.5, Qwen 2.5 & Gemma 2-5x faster with 80% less memory!
+
+
+
+
**Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai)|
+| 💾 **Installation** | [unsloth/README.md](https://github.com/unslothai/unsloth/tree/main#-installation-instructions)|
+| 🥇 **Benchmarking** | [Performance Tables](https://github.com/unslothai/unsloth/tree/main#-performance-benchmarking)
+| 🌐 **Released Models** | [Unsloth Releases](https://docs.unsloth.ai/get-started/all-our-models)|
+| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)|
+| 
**Reddit** | [Join our Reddit page](https://reddit.com/r/unsloth)|
+
+## ⭐ Key Features
+- All kernels written in [OpenAI's Triton](https://openai.com/research/triton) language. **Manual backprop engine**.
+- **0% loss in accuracy** - no approximation methods - all exact.
+- No change of hardware. Supports NVIDIA GPUs since 2018+. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) [Check your GPU!](https://developer.nvidia.com/cuda-gpus) GTX 1070, 1080 works, but is slow.
+- Works on **Linux** and **Windows** via WSL.
+- Supports 4bit and 16bit QLoRA / LoRA finetuning via [bitsandbytes](https://github.com/TimDettmers/bitsandbytes).
+- Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for up to **30x faster training**!
+- If you trained a model with 🦥Unsloth, you can use this cool sticker! 
+
+
+## 🥇 Performance Benchmarking
+- For the full list of **reproducible** benchmarking tables, [go to our website](https://unsloth.ai/blog/mistral-benchmark#Benchmark%20tables)
+
+| 1 A100 40GB | 🤗Hugging Face | Flash Attention | 🦥Unsloth Open Source | 🦥[Unsloth Pro](https://unsloth.ai/pricing) |
+|--------------|--------------|-----------------|---------------------|-----------------|
+| Alpaca | 1x | 1.04x | 1.98x | **15.64x** |
+| LAION Chip2 | 1x | 0.92x | 1.61x | **20.73x** |
+| OASST | 1x | 1.19x | 2.17x | **14.83x** |
+| Slim Orca | 1x | 1.18x | 2.22x | **14.82x** |
+
+- Benchmarking table below was conducted by [🤗Hugging Face](https://huggingface.co/blog/unsloth-trl).
+
+| Free Colab T4 | Dataset | 🤗Hugging Face | Pytorch 2.1.1 | 🦥Unsloth | 🦥 VRAM reduction |
+| --- | --- | --- | --- | --- | --- |
+| Llama-2 7b | OASST | 1x | 1.19x | 1.95x | -43.3% |
+| Mistral 7b | Alpaca | 1x | 1.07x | 1.56x | -13.7% |
+| Tiny Llama 1.1b | Alpaca | 1x | 2.06x | 3.87x | -73.8% |
+| DPO with Zephyr | Ultra Chat | 1x | 1.09x | 1.55x | -18.6% |
+
+
+
+## 💾 Installation Instructions
+
+For stable releases, use `pip install unsloth`. We recommend `pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"` for most installations though.
+
+### Conda Installation
+`⚠️Only use Conda if you have it. If not, use Pip`. Select either `pytorch-cuda=11.8,12.1` for CUDA 11.8 or CUDA 12.1. We support `python=3.10,3.11,3.12`.
+```bash
+conda create --name unsloth_env \
+ python=3.11 \
+ pytorch-cuda=12.1 \
+ pytorch cudatoolkit xformers -c pytorch -c nvidia -c xformers \
+ -y
+conda activate unsloth_env
+
+pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
+pip install --no-deps trl peft accelerate bitsandbytes
+```
+
+
](https://github.com/unslothai/unsloth)
+"""
+
+
+def _determine_username(save_directory, old_username, token):
+ username = ""
+ save_directory = save_directory.lstrip("./")
+ if "/" not in save_directory:
+ from huggingface_hub import whoami
+ try:
+ username = whoami(token = token)["name"]
+ if type(old_username) is str and username != old_username:
+ username = old_username
+ pass
+ save_directory = f"{username}/{save_directory}"
+ except:
+ raise RuntimeError(f"Unsloth: {save_directory} is not a Huggingface directory.")
+ else:
+ username = save_directory.split("/")[0]
+ pass
+ return save_directory, username
+pass
+
+
+def create_huggingface_repo(
+ model,
+ save_directory,
+ token = None,
+ private = False,
+):
+ if token is None :
+ token = get_token()
+ pass
+ save_directory, username = _determine_username(save_directory, "", token)
+
+ from huggingface_hub import create_repo
+ try:
+ create_repo(
+ repo_id = save_directory,
+ token = token,
+ repo_type = "model",
+ exist_ok = False,
+ private = private,
+ )
+
+ # Create model card
+ from huggingface_hub import ModelCard
+ content = MODEL_CARD.format(
+ username = username,
+ base_model = model.config._name_or_path,
+ model_type = model.config.model_type,
+ method = "",
+ extra = "unsloth",
+ )
+ card = ModelCard(content)
+ card.push_to_hub(save_directory, token = token)
+ except:
+ pass
+ hf_api = HfApi(token = token)
+ return save_directory, hf_api
+pass
+
+
+def upload_to_huggingface(
+ model,
+ save_directory,
+ token,
+ method,
+ extra = "",
+ file_location = None,
+ old_username = None,
+ private = None,
+ create_config = True,
+):
+ save_directory, username = _determine_username(save_directory, old_username, token)
+
+ from huggingface_hub import create_repo
+ try:
+ create_repo(
+ repo_id = save_directory,
+ token = token,
+ repo_type = "model",
+ exist_ok = False,
+ private = private,
+ )
+
+ # Create model card
+ from huggingface_hub import ModelCard
+ content = MODEL_CARD.format(
+ username = username,
+ base_model = model.config._name_or_path,
+ model_type = model.config.model_type,
+ method = "",
+ extra = extra,
+ )
+ card = ModelCard(content)
+ card.push_to_hub(save_directory, token = token)
+ except:
+ pass
+
+ if file_location is not None:
+ # Now upload file
+ hf_api = HfApi(token = token)
+
+ if "/" in file_location:
+ uploaded_location = file_location[file_location.rfind("/")+1:]
+ else:
+ uploaded_location = file_location
+ pass
+
+ # find ftevent file from tensorboard and upload it
+ import glob
+ ftevent_files = glob.glob("*out.tfevents*", recursive = True)
+ if len(ftevent_files) > 0:
+ print("Unsloth: Uploading tensorboard files... Please wait...", file_location + "*out.tfevents*")
+ for ftevent_file in ftevent_files:
+ hf_api.upload_file(
+ path_or_fileobj = ftevent_file,
+ path_in_repo = ftevent_file.replace(file_location, ""),
+ repo_id = save_directory,
+ repo_type = "model",
+ commit_message = "(Trained with Unsloth)",
+ )
+ pass
+ pass
+
+ hf_api.upload_file(
+ path_or_fileobj = file_location,
+ path_in_repo = uploaded_location,
+ repo_id = save_directory,
+ repo_type = "model",
+ commit_message = "(Trained with Unsloth)",
+ )
+
+ # We also upload a config.json file
+ if create_config:
+ import json
+ with open("_temporary_unsloth_config.json", "w") as file:
+ json.dump({"model_type" : model.config.model_type}, file, indent = 4)
+ pass
+ hf_api.upload_file(
+ path_or_fileobj = "_temporary_unsloth_config.json",
+ path_in_repo = "config.json",
+ repo_id = save_directory,
+ repo_type = "model",
+ commit_message = "(Trained with Unsloth)",
+ )
+ os.remove("_temporary_unsloth_config.json")
+ pass
+ pass
+ return username
+pass
+
+
+def fix_tokenizer_bos_token(tokenizer):
+ # Check if BOS added already, then warn
+ fix_bos_token = False
+ chat_template = getattr(tokenizer, "chat_template", None)
+
+ if (tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None)):
+ if chat_template is not None and \
+ (
+ tokenizer.bos_token in chat_template or \
+ "{bos_token}" in chat_template.replace(" ", "") or \
+ "{bos_token+" in chat_template.replace(" ", "")
+ ):
+
+ fix_bos_token = True
+ logger.warning(
+ "Unsloth: ##### The current model auto adds a BOS token.\n"\
+ "Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily."
+ )
+
+ # Remove {{bos_token}}
+ new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template)
+ # Remove {{bos_token +
+ new_chat_template = re.sub(r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}", "", new_chat_template)
+
+ tokenizer.chat_template = new_chat_template
+
+ pass
+ pass
+ return fix_bos_token, chat_template
+pass
+
+
+def create_ollama_modelfile(tokenizer, gguf_location):
+ """
+ Creates an Ollama Modelfile.
+ Use ollama.create(model = "new_ollama_model", modelfile = modelfile)
+ """
+ modelfile = getattr(tokenizer, "_ollama_modelfile", None)
+ if modelfile is None: return None
+
+ FILE_LOCATION_REPLACER = "⚫@✅#🦥__FILE_LOCATION__⚡@🦥#⛵"
+ EOS_TOKEN_REPLACER = "⚫@✅#🦥__EOS_TOKEN__⚡@🦥#⛵"
+ LEFT_BRACKET_REPLACER = "⚫@✅#🦥"
+ RIGHT_BRACKET_REPLACER = "⚡@🦥#⛵"
+
+ # Fixes https://github.com/unslothai/unsloth/issues/1087
+ # We must convert all {'s and }'s but keep {__FILE_LOCATION__} intact
+ modelfile = modelfile\
+ .replace("{__FILE_LOCATION__}", FILE_LOCATION_REPLACER)\
+ .replace("{__EOS_TOKEN__}", EOS_TOKEN_REPLACER)\
+ .replace("{", LEFT_BRACKET_REPLACER)\
+ .replace("}", RIGHT_BRACKET_REPLACER)
+
+ # Revert {__FILE_LOCATION__} back
+ modelfile = modelfile\
+ .replace(FILE_LOCATION_REPLACER, "{__FILE_LOCATION__}")\
+ .replace(EOS_TOKEN_REPLACER, "{__EOS_TOKEN__}")
+
+ if "__EOS_TOKEN__" in modelfile:
+ modelfile = modelfile.format(
+ __FILE_LOCATION__ = gguf_location,
+ __EOS_TOKEN__ = tokenizer.eos_token,
+ )
+ else:
+ modelfile = modelfile.format(
+ __FILE_LOCATION__ = gguf_location,
+ )
+ pass
+
+ modelfile = modelfile\
+ .replace("⚫@✅#🦥", "{")\
+ .replace("⚡@🦥#⛵", "}")\
+ .rstrip()
+
+ return modelfile
+pass
+
+
+def unsloth_save_pretrained_gguf(
+ self,
+ save_directory : Union[str, os.PathLike],
+ tokenizer = None,
+ quantization_method : str = "fast_quantized",
+ first_conversion : str = None,
+ push_to_hub : bool = False,
+ token : Optional[Union[str, bool]] = None,
+ private : Optional[bool] = None,
+ is_main_process : bool = True,
+ state_dict : Optional[dict] = None,
+ save_function : Callable = torch.save,
+ max_shard_size : Union[int, str] = "5GB",
+ safe_serialization : bool = True,
+ variant : Optional[str] = None,
+ save_peft_format : bool = True,
+ tags : List[str] = None,
+ temporary_location : str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage : float = 0.85,
+):
+ """
+ Same as .save_pretrained(...) except 4bit weights are auto
+ converted to float16 then converted to GGUF / llama.cpp format.
+
+ Choose for `quantization_method` to be:
+ "not_quantized" : "Recommended. Fast conversion. Slow inference, big files.",
+ "fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.",
+ "quantized" : "Recommended. Slow conversion. Fast inference, small files.",
+ "f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.",
+ "f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.",
+ "q8_0" : "Fast conversion. High resource use, but generally acceptable.",
+ "q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K",
+ "q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K",
+ "q2_k" : "Uses Q4_K for the attention.vw and feed_forward.w2 tensors, Q2_K for the other tensors.",
+ "q3_k_l" : "Uses Q5_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K",
+ "q3_k_m" : "Uses Q4_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K",
+ "q3_k_s" : "Uses Q3_K for all tensors",
+ "q4_0" : "Original quant method, 4-bit.",
+ "q4_1" : "Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.",
+ "q4_k_s" : "Uses Q4_K for all tensors",
+ "q4_k" : "alias for q4_k_m",
+ "q5_k" : "alias for q5_k_m",
+ "q5_0" : "Higher accuracy, higher resource usage and slower inference.",
+ "q5_1" : "Even higher accuracy, resource usage and slower inference.",
+ "q5_k_s" : "Uses Q5_K for all tensors",
+ "q6_k" : "Uses Q8_K for all tensors",
+ "iq2_xxs" : "2.06 bpw quantization",
+ "iq2_xs" : "2.31 bpw quantization",
+ "iq3_xxs" : "3.06 bpw quantization",
+ "q3_k_xs" : "3-bit extra small quantization",
+ """
+ if tokenizer is None:
+ raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.")
+
+ arguments = dict(locals())
+ arguments["model"] = self
+ arguments["tokenizer"] = tokenizer
+ arguments["push_to_hub"] = False # We save ourselves
+ arguments["save_method"] = "merged_16bit" # Must be 16bit
+ del arguments["self"]
+ del arguments["quantization_method"]
+ del arguments["first_conversion"]
+
+ # Fix tokenizer adding an extra BOS token at the front
+ fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
+
+ # Non blocking install GGUF first
+ if not os.path.exists("llama.cpp"):
+
+ if IS_KAGGLE_ENVIRONMENT:
+ # Kaggle is weird - no blocking installs, and no CUDA?
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda = False)
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ python_install.wait()
+ pass
+ else:
+ try:
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ except:
+ # Retry by recloning llama.cpp
+ if IS_KAGGLE_ENVIRONMENT:
+ # Kaggle is weird - no blocking installs, and no CUDA?
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda = False)
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ python_install.wait()
+ pass
+ pass
+ pass
+
+ # Use old chat template if the bos is removed
+ if fix_bos_token:
+ tokenizer.chat_template = old_chat_template
+ pass
+
+ for _ in range(3):
+ gc.collect()
+
+ model_dtype = self.config.torch_dtype
+ model_type = self.config.model_type
+ if type(model_dtype) is str:
+ assert(model_dtype == "float16" or model_dtype == "bfloat16")
+ elif model_dtype == torch.float16:
+ model_dtype = "float16"
+ elif model_dtype == torch.bfloat16:
+ model_dtype = "bfloat16"
+ else:
+ raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
+ pass
+
+ is_sentencepiece_model = check_if_sentencepiece_model(self)
+
+ # Save to GGUF
+ all_file_locations, want_full_precision = save_to_gguf(
+ model_type, model_dtype, is_sentencepiece_model,
+ new_save_directory, quantization_method, first_conversion, makefile,
+ )
+
+ # Save Ollama modelfile
+ modelfile = create_ollama_modelfile(tokenizer, all_file_locations[0])
+ modelfile_location = None
+ if modelfile is not None:
+ modelfile_location = os.path.join(new_save_directory, "Modelfile")
+ with open(modelfile_location, "w") as file:
+ file.write(modelfile)
+ pass
+ print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}")
+ pass
+
+ if fix_bos_token:
+ logger.warning(
+ "Unsloth: ##### The current model auto adds a BOS token.\n"\
+ "Unsloth: ##### We removed it in GGUF's chat template for you."
+ )
+ pass
+
+ if push_to_hub:
+ print("Unsloth: Uploading GGUF to Huggingface Hub...")
+
+ # If not needing full precision, skip the first
+ if not want_full_precision: all_file_locations = all_file_locations[1:]
+
+ for file_location in all_file_locations:
+ username = upload_to_huggingface(
+ self, save_directory, token,
+ "GGUF converted", "gguf", file_location, old_username, private,
+ )
+ link = f"{username}/{new_save_directory.lstrip('/.')}" \
+ if username not in new_save_directory else \
+ new_save_directory.lstrip('/.')
+ print(f"Saved GGUF to https://huggingface.co/{link}")
+ pass
+
+ # Save modelfile
+ if modelfile_location is not None:
+ username = upload_to_huggingface(
+ self, save_directory, token,
+ "GGUF converted", "gguf", modelfile_location, old_username, private,
+ )
+ print(f"Saved Ollama Modelfile to https://huggingface.co/{link}")
+ pass
+ pass
+pass
+
+
+def unsloth_push_to_hub_gguf(
+ self,
+ repo_id : str,
+ tokenizer = None,
+ quantization_method : str = "fast_quantized",
+ first_conversion : str = None,
+ use_temp_dir : Optional[bool] = None,
+ commit_message : Optional[str] = "Trained with Unsloth",
+ private : Optional[bool] = None,
+ token : Union[bool, str, None] = None,
+ max_shard_size : Union[int, str, None] = "5GB",
+ create_pr : bool = False,
+ safe_serialization : bool = True,
+ revision : str = None,
+ commit_description : str = "Upload model trained with Unsloth 2x faster",
+ tags : Optional[List[str]] = None,
+ temporary_location : str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage : float = 0.85,
+):
+ """
+ Same as .push_to_hub(...) except 4bit weights are auto
+ converted to float16 then converted to GGUF / llama.cpp format.
+
+ Choose for `quantization_method` to be:
+ "not_quantized" : "Recommended. Fast conversion. Slow inference, big files.",
+ "fast_quantized" : "Recommended. Fast conversion. OK inference, OK file size.",
+ "quantized" : "Recommended. Slow conversion. Fast inference, small files.",
+ "f32" : "Not recommended. Retains 100% accuracy, but super slow and memory hungry.",
+ "f16" : "Fastest conversion + retains 100% accuracy. Slow and memory hungry.",
+ "q8_0" : "Fast conversion. High resource use, but generally acceptable.",
+ "q4_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q4_K",
+ "q5_k_m" : "Recommended. Uses Q6_K for half of the attention.wv and feed_forward.w2 tensors, else Q5_K",
+ "q2_k" : "Uses Q4_K for the attention.vw and feed_forward.w2 tensors, Q2_K for the other tensors.",
+ "q3_k_l" : "Uses Q5_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K",
+ "q3_k_m" : "Uses Q4_K for the attention.wv, attention.wo, and feed_forward.w2 tensors, else Q3_K",
+ "q3_k_s" : "Uses Q3_K for all tensors",
+ "q4_0" : "Original quant method, 4-bit.",
+ "q4_1" : "Higher accuracy than q4_0 but not as high as q5_0. However has quicker inference than q5 models.",
+ "q4_k_s" : "Uses Q4_K for all tensors",
+ "q5_0" : "Higher accuracy, higher resource usage and slower inference.",
+ "q5_1" : "Even higher accuracy, resource usage and slower inference.",
+ "q5_k_s" : "Uses Q5_K for all tensors",
+ "q6_k" : "Uses Q8_K for all tensors",
+ """
+ if tokenizer is None:
+ raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.")
+
+ arguments = dict(locals())
+ arguments["model"] = self
+ arguments["tokenizer"] = tokenizer
+ arguments["save_directory"] = repo_id
+ arguments["push_to_hub"] = False # We save ourselves
+ arguments["save_method"] = "merged_16bit" # Must be 16bit
+ del arguments["self"]
+ del arguments["repo_id"]
+ del arguments["quantization_method"]
+ del arguments["first_conversion"]
+
+ # Fix tokenizer adding an extra BOS token at the front
+ fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
+
+ # Non blocking install GGUF first
+ if not os.path.exists("llama.cpp"):
+
+ if IS_KAGGLE_ENVIRONMENT:
+ # Kaggle is weird - no blocking installs, and no CUDA?
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda = False)
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ python_install.wait()
+ pass
+ else:
+ try:
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ except:
+ # Retry by recloning llama.cpp
+ if IS_KAGGLE_ENVIRONMENT:
+ # Kaggle is weird - no blocking installs, and no CUDA?
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda = False)
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["gguf", "protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ new_save_directory, old_username = unsloth_save_model(**arguments)
+ python_install.wait()
+ pass
+ pass
+ pass
+
+ # Use old chat template if the bos is removed
+ if fix_bos_token:
+ tokenizer.chat_template = old_chat_template
+ pass
+
+ for _ in range(3):
+ gc.collect()
+
+ model_dtype = self.config.torch_dtype
+ model_type = self.config.model_type
+ if type(model_dtype) is str:
+ assert(model_dtype == "float16" or model_dtype == "bfloat16")
+ elif model_dtype == torch.float16:
+ model_dtype = "float16"
+ elif model_dtype == torch.bfloat16:
+ model_dtype = "bfloat16"
+ else:
+ raise TypeError("Unsloth: Model dtype can only be float16 or bfloat16")
+ pass
+
+ is_sentencepiece_model = check_if_sentencepiece_model(self)
+
+ # Save to GGUF
+ all_file_locations, want_full_precision = save_to_gguf(
+ model_type, model_dtype, is_sentencepiece_model,
+ new_save_directory, quantization_method, first_conversion, makefile,
+ )
+
+ # Save Ollama modelfile
+ modelfile = create_ollama_modelfile(tokenizer, all_file_locations[0])
+ modelfile_location = None
+ if modelfile is not None:
+ modelfile_location = os.path.join(new_save_directory, "Modelfile")
+ with open(modelfile_location, "w") as file:
+ file.write(modelfile)
+ pass
+ print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}")
+ pass
+
+ # If not needing full precision, skip the first
+ if not want_full_precision: all_file_locations = all_file_locations[1:]
+
+ for file_location in all_file_locations:
+ print("Unsloth: Uploading GGUF to Huggingface Hub...")
+ username = upload_to_huggingface(
+ self, repo_id, token,
+ "GGUF converted", "gguf", file_location, old_username, private,
+ )
+ link = f"{username}/{new_save_directory.lstrip('/.')}" \
+ if username not in new_save_directory else \
+ new_save_directory.lstrip('/.')
+
+ print(f"Saved GGUF to https://huggingface.co/{link}")
+ pass
+
+ # Save modelfile
+ if modelfile_location is not None:
+ username = upload_to_huggingface(
+ self, repo_id, token,
+ "GGUF converted", "gguf", modelfile_location, old_username, private,
+ )
+ print(f"Saved Ollama Modelfile to https://huggingface.co/{link}")
+ pass
+
+ if fix_bos_token:
+ logger.warning(
+ "Unsloth: ##### The current model auto adds a BOS token.\n"\
+ "Unsloth: ##### We removed it in GGUF's chat template for you."
+ )
+ pass
+pass
+
+# Corrected function to save LoRA to a custom directory
+def save_lora_to_custom_dir(model, tokenizer, save_directory):
+ # Create the custom directory if it doesn't exist
+ os.makedirs(save_directory, exist_ok=True)
+
+ # Call the unsloth_save_model function with the custom directory
+ unsloth_save_model(
+ model,
+ tokenizer,
+ save_directory=save_directory,
+ save_method="lora",
+ push_to_hub=False,
+ )
+
+# Corrected method within the model class to convert LoRA to GGML and push to Hugging Face Hub
+def unsloth_convert_lora_to_ggml_and_push_to_hub(
+ self,
+ tokenizer,
+ repo_id: str,
+ use_temp_dir: Optional[bool] = None,
+ commit_message: Optional[str] = "Converted LoRA to GGML with Unsloth",
+ private: Optional[bool] = None,
+ token: Union[bool, str, None] = None,
+ create_pr: bool = False,
+ revision: str = None,
+ commit_description: str = "Convert LoRA to GGML format using Unsloth",
+ temporary_location: str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage: float = 0.85,
+):
+ if not os.path.exists("llama.cpp"):
+ if IS_KAGGLE_ENVIRONMENT:
+ python_install = install_python_non_blocking(["protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda=False)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ python_install.wait()
+ else:
+ makefile = None
+
+ for _ in range(3):
+ gc.collect()
+
+ lora_directory_push = "lora-to-ggml-push"
+ save_lora_to_custom_dir(self, tokenizer, lora_directory_push)
+
+ model_type = self.config.model_type
+ output_file = os.path.join(lora_directory_push, "ggml-adapter-model.bin")
+
+ print(f"Unsloth: Converting auto-saved LoRA adapters at {lora_directory_push} to GGML format.")
+ print(f"The output file will be {output_file}")
+
+ command = f"python3 llama.cpp/convert-lora-to-ggml.py {lora_directory_push} {output_file} llama"
+
+ try:
+ with subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) as sp:
+ for line in sp.stdout:
+ print(line, end="", flush=True)
+ for line in sp.stderr:
+ print(line, end="", flush=True)
+ sp.wait()
+ if sp.returncode != 0:
+ raise subprocess.CalledProcessError(sp.returncode, command)
+ except subprocess.CalledProcessError as e:
+ print(f"Error: Conversion failed with return code {e.returncode}")
+ return
+
+ print(f"Unsloth: Conversion completed! Output file: {output_file}")
+
+ print("Unsloth: Uploading GGML file to Hugging Face Hub...")
+ username = upload_to_huggingface(
+ self, repo_id, token,
+ "GGML converted LoRA", "ggml", output_file, None, private,
+ )
+ link = f"{repo_id.lstrip('/')}"
+ print("Unsloth: Done.")
+ print(f"Converted LoRA to GGML and uploaded to https://huggingface.co/{link}")
+ print("\nThis GGML making function was made by Maheswar. Ping him @Maheswar on the Unsloth Discord or on HuggingFace (@mahiatlinux) if you like this!")
+
+def unsloth_convert_lora_to_ggml_and_save_locally(
+ self,
+ save_directory: str, # Added parameter for the folder name
+ tokenizer,
+ temporary_location: str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage: float = 0.85,
+):
+ if not os.path.exists("llama.cpp"):
+ if IS_KAGGLE_ENVIRONMENT:
+ python_install = install_python_non_blocking(["protobuf"])
+ python_install.wait()
+ install_llama_cpp_blocking(use_cuda=False)
+ makefile = None
+ else:
+ git_clone = install_llama_cpp_clone_non_blocking()
+ python_install = install_python_non_blocking(["protobuf"])
+ git_clone.wait()
+ makefile = install_llama_cpp_make_non_blocking()
+ python_install.wait()
+ else:
+ makefile = None
+
+ for _ in range(3):
+ gc.collect()
+
+ # Use the provided save_directory for local saving
+ save_lora_to_custom_dir(self, tokenizer, save_directory)
+
+ model_type = self.config.model_type
+ output_file = os.path.join(save_directory, "ggml-adapter-model.bin")
+
+ print(f"Unsloth: Converting auto-saved LoRA adapters at {save_directory} to GGML format.")
+ print(f"The output file will be {output_file}")
+
+ command = f"python3 llama.cpp/convert-lora-to-ggml.py {save_directory} {output_file} llama"
+
+ try:
+ with subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) as sp:
+ for line in sp.stdout:
+ print(line, end="", flush=True)
+ for line in sp.stderr:
+ print(line, end="", flush=True)
+ sp.wait()
+ if sp.returncode != 0:
+ raise subprocess.CalledProcessError(sp.returncode, command)
+ except subprocess.CalledProcessError as e:
+ print(f"Error: Conversion failed with return code {e.returncode}")
+ return
+ print("Unsloth: Done.")
+ print(f"Unsloth: Conversion completed! Output file: {output_file}")
+ print("\nThis GGML making function was made by Maheswar. Ping him @Maheswar on the Unsloth Discord or on HuggingFace (@mahiatlinux) if you like this!")
+pass
+
+
+from .models.loader_utils import get_model_name
+from unsloth_zoo.saving_utils import merge_and_overwrite_lora
+
+@torch.inference_mode
+def unsloth_generic_save(
+ model,
+ tokenizer,
+ save_directory : Union[str, os.PathLike] = "unsloth_finetuned_merge",
+ save_method : str = "lora", # ["lora", "merged_16bit", "merged_4bit"]
+ push_to_hub : bool = False,
+ token : Optional[Union[str, bool]] = None,
+ is_main_process : bool = True,
+ state_dict : Optional[dict] = None,
+ save_function : Callable = torch.save,
+ max_shard_size : Union[int, str] = "5GB",
+ safe_serialization : bool = True,
+ variant : Optional[str] = None,
+ save_peft_format : bool = True,
+
+ # Push to hub
+ use_temp_dir : Optional[bool] = None,
+ commit_message : Optional[str] = "Trained with Unsloth",
+ private : Optional[bool] = None,
+ create_pr : bool = False,
+ revision : str = None,
+ commit_description : str = "Upload model trained with Unsloth 2x faster",
+ tags : List[str] = None,
+
+ # Our functions
+ temporary_location : str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage : float = 0.9,
+):
+ if token is None and push_to_hub: token = get_token()
+ merge_and_overwrite_lora(
+ get_model_name,
+ model = model,
+ tokenizer = tokenizer,
+ save_directory = save_directory,
+ push_to_hub = push_to_hub,
+ private = private,
+ token = token,
+ output_dtype = None,
+ low_disk_space_usage = False,
+ use_temp_file = False,
+ )
+ return
+pass
+
+
+def unsloth_generic_save_pretrained_merged(
+ self,
+ save_directory : Union[str, os.PathLike],
+ tokenizer = None,
+ save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"]
+ push_to_hub : bool = False,
+ token : Optional[Union[str, bool]] = None,
+ is_main_process : bool = True,
+ state_dict : Optional[dict] = None,
+ save_function : Callable = torch.save,
+ max_shard_size : Union[int, str] = "5GB",
+ safe_serialization : bool = True,
+ variant : Optional[str] = None,
+ save_peft_format : bool = True,
+ tags : List[str] = None,
+ temporary_location : str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage : float = 0.75,
+):
+ """
+ Same as .push_to_hub(...) except 4bit weights are auto
+ converted to float16 with as few overhead as possible.
+
+ Choose for `save_method` to be either:
+ 1. `16bit`: Merge LoRA into float16 weights. Useful for GGUF / llama.cpp.
+ 2. `4bit`: Merge LoRA into int4 weights. Useful for DPO / HF inference.
+ 3. `lora`: Save LoRA adapters with no merging. Useful for HF inference.
+ """
+ if tokenizer is None:
+ logger.warning_once(
+ "Unsloth: You're not saving a tokenizer as well?\n"\
+ "You can do it separately via `tokenizer.save_pretrained(...)`"
+ )
+ pass
+
+ arguments = dict(locals())
+ arguments["model"] = self
+ del arguments["self"]
+ unsloth_generic_save(**arguments)
+ for _ in range(3):
+ gc.collect()
+pass
+
+
+def unsloth_generic_push_to_hub_merged(
+ self,
+ repo_id : str,
+ tokenizer = None,
+ save_method : str = "merged_16bit", # ["lora", "merged_16bit", "merged_4bit"]
+ use_temp_dir : Optional[bool] = None,
+ commit_message : Optional[str] = "Trained with Unsloth",
+ private : Optional[bool] = None,
+ token : Union[bool, str, None] = None,
+ max_shard_size : Union[int, str, None] = "5GB",
+ create_pr : bool = False,
+ safe_serialization : bool = True,
+ revision : str = None,
+ commit_description : str = "Upload model trained with Unsloth 2x faster",
+ tags : Optional[List[str]] = None,
+ temporary_location : str = "_unsloth_temporary_saved_buffers",
+ maximum_memory_usage : float = 0.75,
+):
+ """
+ Same as .push_to_hub(...) except 4bit weights are auto
+ converted to float16 with as few overhead as possible.
+
+ Choose for `save_method` to be either:
+ 1. `16bit`: Merge LoRA into float16 weights. Useful for GGUF / llama.cpp.
+ 2. `4bit`: Merge LoRA into int4 weights. Useful for DPO / HF inference.
+ 3. `lora`: Save LoRA adapters with no merging. Useful for HF inference.
+ """
+ if tokenizer is None:
+ logger.warning_once(
+ "Unsloth: You're not saving a tokenizer as well?\n"\
+ "You can do it separately via `tokenizer.push_to_hub(...)`"
+ )
+ pass
+
+ arguments = dict(locals())
+ arguments["model"] = self
+ arguments["save_directory"] = repo_id
+ arguments["push_to_hub"] = True
+ del arguments["self"]
+ del arguments["repo_id"]
+ unsloth_generic_save(**arguments)
+ for _ in range(3):
+ gc.collect()
+pass
+
+
+def not_implemented_save(*args, **kwargs):
+ raise NotImplementedError("Unsloth: Sorry GGUF is currently not supported for vision models!")
+pass
+
+
+def patch_saving_functions(model, vision = False):
+ import inspect
+ import types
+ from typing import Callable, Optional, Union, List
+
+ # And now re add our saving methods!
+ if model.push_to_hub.__name__ == "unsloth_push_to_hub":
+ original_push_to_hub = model.original_push_to_hub
+ else:
+ original_push_to_hub = model.push_to_hub
+ pass
+
+ signature = str(inspect.signature(original_push_to_hub)).replace("NoneType", "None")
+ signature = signature[1:]
+ signature = re.sub("