# Quantization Notes:
bpw: 5
hb: 6
calibration_length: 8192
measurement_length: 8192
## Quantization Code:
Posting this here for convenience in case anyone is interested or finds it useful. I run this code using a conda 3.12 python env in WSL 2 Ubuntu. Steps to run include creating conda env and installing / upgrading exllamav2, logging into huggingface using the "huggingface-cli login" terminal command, configuring the config.yaml file, then running the python script.
```config.yaml
base_model_name: "Endurance-100B-v1"
input_model: "~/models/TheDrummer_Endurance-100B-v1"
download_output_dir: "~/models"
output_base_path: "~/models/exl2-converted"
hf_username: "NobodySpecial"
default_hb: 6 # Default head bits value
exllama_path: "~/exllamav2"
quantizations:
- bpw: 5
calibration_length: 8192 # Optional: specify calibration length in tokens
measurement_length: 8192 # Optional: specify measurement length in tokens
- bpw: 6
hb: 8 # Optional
calibration_length: 8192 # Optional: specify calibration length in tokens
measurement_length: 8192 # Optional: specify measurement length in tokens
```
```script.py
import yaml
import os
import sys
import subprocess
import logging
import re
from tqdm import tqdm
from pathlib import Path
from huggingface_hub import HfApi, create_repo, login, hf_hub_download
# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def run_command(command_list, timeout=300):
try:
process = subprocess.Popen(
command_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
universal_newlines=True
)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
logging.info(output.strip())
rc = process.poll()
if rc != 0:
error_output = process.stderr.read()
logging.error(f"Error executing command: {' '.join(command_list)}")
logging.error(f"Error output: {error_output}")
return False
logging.info(f"Command executed successfully: {' '.join(command_list)}")
return True
except subprocess.TimeoutExpired:
logging.error(f"Command timed out: {' '.join(command_list)}")
process.kill()
return False
except Exception as e:
logging.error(f"Unexpected error executing command: {' '.join(command_list)}")
logging.error(f"Error: {str(e)}")
return False
def validate_config(config):
required_keys = [
'exllama_path',
'base_model_name',
'input_model',
'output_base_path',
'hf_username',
'quantizations'
]
missing_keys = [key for key in required_keys if key not in config]
if missing_keys:
logging.error(f"Missing required configuration keys: {', '.join(missing_keys)}")
return False
# Validate exllama_path
if not os.path.isdir(os.path.expanduser(config['exllama_path'])):
logging.error(f"exllama_path does not exist or is not a directory: {config['exllama_path']}")
return False
# Validate output_base_path
output_base_path = os.path.expanduser(config['output_base_path'])
if not os.path.isdir(output_base_path):
try:
os.makedirs(output_base_path, exist_ok=True)
logging.info(f"Created output_base_path directory: {output_base_path}")
except OSError as e:
logging.error(f"Failed to create output_base_path directory: {str(e)}")
return False
return True
def authenticate_hf():
try:
# Read the token from the local cache file
token_path = os.path.expanduser("~/.cache/huggingface/token")
with open(token_path, "r") as token_file:
hf_token = token_file.read().strip()
# Use the token to login
login(token=hf_token)
logging.info("Authenticated with Hugging Face successfully.")
except Exception as e:
logging.error(f"Failed to authenticate with Hugging Face: {str(e)}")
return False
return True
def sanitize_model_and_branch_names(model, branch):
# Remove trailing slash if present
model = model.rstrip('/')
# Remove base URL if present
if model.startswith("https://huggingface.co/"):
model = model[len("https://huggingface.co/"):]
# Split model and branch if provided in model name
model_parts = model.split(":")
model = model_parts[0]
branch = model_parts[1] if len(model_parts) > 1 else branch
# Use 'main' as default branch if not specified
if branch is None:
branch = "main"
# Validate branch name
if not re.match(r"^[a-zA-Z0-9._-]+$", branch):
raise ValueError("Invalid branch name. Only alphanumeric characters, period, underscore and dash are allowed.")
return model, branch
def download_model(model_name, branch_name, output_dir):
# Sanitize model and branch names
model_name, branch_name = sanitize_model_and_branch_names(model_name, branch_name)
# Expand user directory if needed
output_dir = os.path.expanduser(output_dir)
# Initialize Hugging Face API
api = HfApi()
# Create output directory
output_folder = Path(output_dir) / f"{'_'.join(model_name.split('/')[-2:])}"
if branch_name != "main":
output_folder = output_folder.with_name(f"{output_folder.name}_{branch_name}")
output_folder.mkdir(parents=True, exist_ok=True)
# Get file list
try:
files = api.list_repo_files(model_name, revision=branch_name)
except Exception as e:
logging.error(f"Error accessing repository: {e}")
return None
# Download files
for file in tqdm(files, desc="Downloading files"):
try:
hf_hub_download(
repo_id=model_name,
filename=file,
revision=branch_name,
local_dir=output_folder,
local_dir_use_symlinks=False
)
except Exception as e:
logging.error(f"Error downloading {file}: {e}")
logging.info(f"Model downloaded to {output_folder}")
return output_folder
def resolve_input_model(config):
input_model = os.path.expanduser(config['input_model'])
if os.path.isdir(input_model):
# Input model is a local directory
logging.info(f"Using local model directory: {input_model}")
return input_model
else:
# Input model is a Hugging Face repository
logging.info(f"Input model is a Hugging Face model: {input_model}")
download_output_dir = os.path.expanduser(config.get('download_output_dir', './models'))
if not os.path.isdir(download_output_dir):
try:
os.makedirs(download_output_dir, exist_ok=True)
logging.info(f"Created download_output_dir directory: {download_output_dir}")
except OSError as e:
logging.error(f"Failed to create download_output_dir directory: {str(e)}")
sys.exit(1)
model_name, branch_name = sanitize_model_and_branch_names(input_model, branch=None)
output_folder = download_model(model_name, branch_name, download_output_dir)
if output_folder is None:
logging.error("Failed to download the model.")
sys.exit(1)
return str(output_folder)
def quantize_and_upload(config, input_model_path):
exllama_path = os.path.expanduser(config['exllama_path'])
base_model_name = config['base_model_name']
output_base_path = os.path.expanduser(config['output_base_path'])
hf_username = config['hf_username']
default_hb = config.get('default_hb', 8)
for quant_config in config['quantizations']:
if 'bpw' not in quant_config:
logging.warning("Skipping quantization config without 'bpw'.")
continue
bpw = quant_config['bpw']
hb = quant_config.get('hb', default_hb)
calibration_length = quant_config.get('calibration_length', 2048)
measurement_length = quant_config.get('measurement_length', calibration_length)
if not isinstance(calibration_length, int) or not isinstance(measurement_length, int):
logging.error(f"Invalid calibration_length or measurement_length values. Expected integers.")
continue
if calibration_length <= 0 or measurement_length <= 0:
logging.error(f"Invalid calibration_length or measurement_length values. Must be positive integers.")
continue
quant_name = f"{base_model_name}-exl2-{bpw}bpw"
work_dir = os.path.join(output_base_path, base_model_name, f"{quant_name}-work")
output_dir = os.path.join(output_base_path, base_model_name, quant_name)
try:
os.makedirs(work_dir, exist_ok=True)
os.makedirs(output_dir, exist_ok=True)
logging.info(f"Directories created or already exist: {work_dir}, {output_dir}")
except OSError as e:
logging.error(f"Failed to create directories for {quant_name}: {str(e)}")
continue
# Run quantization
command_list = [
"python", os.path.join(exllama_path, "convert.py"),
"-i", input_model_path,
"-o", work_dir,
"-cf", output_dir,
"-b", str(bpw),
"-hb", str(hb),
"-l", str(calibration_length),
"-ml", str(measurement_length)
]
if not run_command(command_list):
logging.error(f"Quantization failed for {quant_name}. Skipping upload.")
continue
logging.info(f"Quantization completed for {quant_name}")
# Try to upload to Hugging Face
repo_name = f"{hf_username}/{quant_name}"
try:
create_repo(repo_name, repo_type="model", exist_ok=True)
logging.info(f"Repository '{repo_name}' is ready.")
api = HfApi()
api.upload_folder(
folder_path=output_dir,
repo_id=repo_name,
repo_type="model"
)
logging.info(f"Successfully uploaded {quant_name} to Hugging Face")
except Exception as e:
logging.error(f"Failed to upload {quant_name} to Hugging Face: {str(e)}")
logging.info(f"Quantized model is still available locally at {output_dir}")
logging.info(f"Completed processing for {quant_name}")
if __name__ == "__main__":
config_path = "config.yaml"
try:
with open(config_path, "r") as f:
config = yaml.safe_load(f)
logging.info(f"Configuration loaded from {config_path}")
except yaml.YAMLError as e:
logging.error(f"Error parsing {config_path}: {str(e)}")
sys.exit(1)
except FileNotFoundError:
logging.error(f"{config_path} not found. Please create a config file.")
sys.exit(1)
if not validate_config(config):
logging.error("Configuration validation failed. Exiting.")
sys.exit(1)
if not authenticate_hf():
logging.error("Hugging Face authentication failed. Exiting.")
sys.exit(1)
input_model_path = resolve_input_model(config)
if not input_model_path:
logging.error("Failed to resolve input model path. Exiting.")
sys.exit(1)
quantize_and_upload(config, input_model_path)
logging.info("Script execution completed.")
```
---
base_model:
- TheDrummer/Lazarus-2407-100B
---
# Join our Discord! https://discord.gg/Nbv9pQ88Xb
## 2500+ members strong 💪 Now with more channels! A hub for users and makers alike!
---
# Endurance 100B v1 🎡
*A finetune of [Lazarus 2407 100B](https://huggingface.co/TheDrummer/Lazarus-2407-100B), a pruned Mistral Large 2407 123B!*
> [Do not go gentle into that good night. Rage, rage against the dying of the light!](https://www.youtube.com/watch?v=ESWzPhZWYeI)
---
## Links
- Original: https://huggingface.co/TheDrummer/Endurance-100B-v1
- GGUF: https://huggingface.co/TheDrummer/Endurance-100B-v1-GGUF
- iMatrix: https://huggingface.co/bartowski/Endurance-100B-v1-GGUF (recommended for smaller quants)
## Arsenal (Supported Chat Templates)
- Metharme (Pygmalion in ST)
- Creative, unhinged, unique
### Favorite RP Format
`*action* Dialogue *thoughts* Dialogue *narration*` in 1st person PoV
### Favorite Card
[Audrey by thecooler](https://characterhub.org/characters/thecooler/audrey-5f995ef8)
---
## Technical Details
*Refer to [Lazarus 2407 100B](https://huggingface.co/TheDrummer/Lazarus-2407-100B) for pruning details.*
Endurance used the same hyperparameters as Behemoth. Training loss indicates that they are exactly the same albeit with lower confidence.
Notes on Lazarus 100B (base model for Endurance):
- Testers have noted that 100B seemed nearly identical to 123B.
- One tester said that only one minor mistake was made by the model, requiring a rewrite for failing to pick up on the nuance.
- Another tester went through a satisfying 32K playthrough without any issues.
Endurance 100B v1.0 has gone through additional RP & Instruct training.