ConfidentialMind 🚀🧠
Generative AI Software Infrastructure Simplified 🎉
🔥 Quantized Model: Arcee-Blitz_gptq_g32_4bit 🦾 🔥
Model Details
- Original Model: arcee-ai/Arcee-Blitz
- Quantized Model: Arcee-Blitz_gptq_g32_4bit (this repository)
- Quantization Method: GPTQ (4-bit, group size 32)
- Quantization Library: GPTQModel
- Calibration Dataset: neuralmagic/LLM_compression_calibration (using 1536 samples with seq len 6144)
- Quantized by: ConfidentialMind.com
Usage
from gptqmodel import GPTQModel
from transformers import AutoTokenizer
# Use the local directory or JustJaro/Arcee-Blitz_gptq_g32_4bit after upload
quantized_model_id = "/home/jaro/models/quantized/Arcee-Blitz_gptq_g32_4bit" # or "JustJaro/Arcee-Blitz_gptq_g32_4bit"
tokenizer = AutoTokenizer.from_pretrained(quantized_model_id)
model = GPTQModel.load(quantized_model_id, device="cuda:0") # or "cpu"
input_text = "This is a test prompt"
inputs = tokenizer(input_text, return_tensors="pt").to("cuda:0")
outputs = model.generate(**inputs)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
Package Versions and Installation Instructions
See pyproject.toml for the exact UV project file. See the GPTQModel repo for more details on how to install the package.
Use the provided pyproject.toml:
uv venv
source venv/bin/activate
uv sync
Quantization Script
Below is the exact quantize.py script used to generate this model:
#!/usr/bin/env python3
"""
This script loads a source Hugging Face model and a calibration dataset,
quantizes the model using GPTQModel (with 4-bit precision and a dynamic group size),
saves the quantized model with Transformers’ safe serialization under ~/models/quantized/,
and then creates/updates a Hugging Face repository by uploading the model, tokenizer,
and an auto–generated README.md that includes proper foldable sections, badges, and warnings.
Usage example:
python quantize.py --source-model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
--calibration-dataset wikitext/wikitext-2-raw-v1 \
--seq-len 1024 --nsamples 256 --hf-token <YOUR_HF_TOKEN>
"""
import os
import random
import shutil
import subprocess
from enum import Enum
from pathlib import Path
from typing import List
import torch
import typer
from datasets import load_dataset
from dotenv import load_dotenv, find_dotenv
from gptqmodel import GPTQModel, QuantizeConfig
from gptqmodel.utils import Perplexity
# For later pushing to the model hub
from huggingface_hub import HfApi
from transformers import AutoTokenizer, PreTrainedTokenizerBase
load_dotenv(find_dotenv())
HF_TOKEN = os.getenv("HF_TOKEN")
app = typer.Typer()
class GroupSize(str, Enum):
accurate: int = 32
balanced: int = 64
fast: int = 128
def get_text_from_example(example: dict) -> str:
"""
Returns text from a dataset example.
If the example contains a "text" field, that text is used.
Otherwise, if it has a "messages" field (a list of dicts with a "content" key),
the contents of all messages are concatenated.
"""
if "text" in example and example["text"]:
return example["text"]
elif "messages" in example:
contents = [msg.get("content", "").strip() for msg in example["messages"]]
return " ".join([s for s in contents if s])
else:
return ""
def get_calibration_dataset(
tokenizer: PreTrainedTokenizerBase,
nsamples: int,
seqlen: int,
calibration_dataset: str
) -> List[dict]:
"""
Loads and tokenizes a calibration dataset from the HF Hub (or a local file).
Only examples with at least 80% of seqlen characters (after extraction) are kept.
"""
ds = None
try:
try:
if "/" in calibration_dataset:
parts = calibration_dataset.split("/", 1)
ds = load_dataset(parts[0], parts[1], split="train")
else:
ds = load_dataset(calibration_dataset, split="train")
except Exception as e:
print(f"Error loading dataset '{calibration_dataset}' via load_dataset: {e}")
ds = load_dataset(calibration_dataset, split="train")
print(f"Loaded calibration dataset from full remote path {calibration_dataset}.")
except Exception as e:
print(f"Error loading dataset '{calibration_dataset}' via load_dataset: {e}")
if os.path.exists(calibration_dataset):
try:
ds = load_dataset("json", data_files=calibration_dataset, split="train")
print(f"Loaded calibration dataset from local file {calibration_dataset}.")
except Exception as e2:
print(f"Error loading local json dataset from '{calibration_dataset}': {e2}")
return []
else:
return []
print(f"Dataset features: {ds.features}")
ds = ds.filter(lambda x: len(get_text_from_example(x)) <= int(seqlen * 0.8))
sample_range = min(nsamples, len(ds))
calibration_data = []
for i in range(sample_range):
example = ds[i]
text = get_text_from_example(example)
tokenized = tokenizer(text, truncation=True, max_length=seqlen, return_tensors="pt")
tokenized = {k: v.squeeze(0) for k, v in tokenized.items()}
calibration_data.append(tokenized)
return calibration_data
def calculate_avg_ppl(model, tokenizer):
"""
Computes the average perplexity on the wikitext-2-raw-v1 training split.
"""
ppl = Perplexity(
model=model,
tokenizer=tokenizer,
dataset_path="wikitext",
dataset_name="wikitext-2-raw-v1",
split="train",
text_column="text",
)
ppl_values = ppl.calculate(n_ctx=512, n_batch=512)
avg = sum(ppl_values) / len(ppl_values)
return avg
def get_pinned_package_versions():
"""
Retrieves pinned package versions via 'uv pip freeze'.
"""
try:
result = subprocess.run(["uv", "pip", "freeze"], capture_output=True, text=True, check=True)
packages_output = result.stdout.strip()
versions = {}
for line in packages_output.splitlines():
if "==" in line:
package_name, package_version = line.split("==", 1)
versions[package_name.lower()] = package_version
return versions
except subprocess.CalledProcessError as e:
typer.echo(f"Error running 'uv pip freeze': {e}", err=True)
return {}
except FileNotFoundError:
typer.echo("uv command not found. Make sure uv is installed and in your PATH.", err=True)
return {}
def prepare_model_dir(model_dir: str):
"""Removes the given directory if it exists and creates a new one."""
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
os.makedirs(model_dir, exist_ok=True)
def self_read_script():
"""Returns the full text of this script."""
try:
script_path = os.path.abspath(__file__)
with open(script_path, "r") as f:
script_content = f.read()
except Exception as e:
script_content = "Error reading script content: " + str(e)
return script_content
def get_my_user(hf_token):
"""Retrieves your Hugging Face username from your token."""
api = HfApi(token=hf_token)
user_info = api.whoami()
try:
username = user_info.get("name") or user_info.get("username")
except Exception as e:
typer.echo(f"Error retrieving username from Hugging Face API: {e}. Using default username.")
username = api.whoami()
if not username:
typer.echo("Could not determine your Hugging Face username from the token. Using default username.", err=True)
username = "JustJaro"
return username
def generate_readme(calibration_dataset, nsamples, quantized_model_dir, quantized_model_name,
script_content, seq_len, source_model, username, avg_ppl, group_size_int):
"""
Creates a README.md with dynamic sections:
• A front matter section with badges/links.
• A title that includes a randomly chosen emoji.
• A warning if the perplexity is too high (>30).
• Collapsible sections (using <details>) for model details, usage, installation, script,
quantization performance, disclaimer, contact, license, author and acknowledgements.
• Additional TODO items.
"""
# pick a random emoji from the list (for each model, so it varies)
chosen_emoji = random.choice(["⚡️", "🐣", "🦾", "🤖", "🧠", "🧐", "🚀"])
# Warning if average perplexity is above 30
if avg_ppl > 30:
warning_text = f"\n**⚠️ WARNING: High Perplexity Detected!** The average perplexity is {avg_ppl:.2f}, which exceeds the recommended threshold.\n"
else:
warning_text = ""
# Front matter with badges and links
front_matter = (
"---\n"
'company: "ConfidentialMind"\n'
'emoji: "🧠"\n'
'colorFrom: "blue"\n'
'colorTo: "purple"\n'
"pinned: true\n"
'authors: "JustJaro"\n'
"---\n\n"
"# ConfidentialMind 🚀🧠\n\n"
"Generative AI Software Infrastructure Simplified 🎉\n\n"
"[](https://confidentialmind.com) \n"
"[](mailto:[email protected])\n\n"
)
# Title plus warning (if any)
title = f"# 🔥 Quantized Model: {quantized_model_name} {chosen_emoji} 🔥\n{warning_text}\n"
# Collapsible sections using <details> tags:
model_details_section = f"""<details>
<summary><strong>Model Details</strong></summary>
- **Original Model:** [{source_model}](https://huggingface.co/{source_model})
- **Quantized Model:** {quantized_model_name} (this repository)
- **Quantization Method:** GPTQ (4-bit, group size {group_size_int})
- **Quantization Library:** [GPTQModel](https://github.com/ModelCloud/GPTQModel/tree/main)
- **Calibration Dataset:** {calibration_dataset} (using {nsamples} samples with seq len {seq_len})
- **Quantized by:** [ConfidentialMind.com](https://www.confidentialmind.com)
usage_section = f"""<details>
from gptqmodel import GPTQModel
from transformers import AutoTokenizer
# Use the local directory or {username}/{quantized_model_name} after upload
quantized_model_id = "{quantized_model_dir}" # or "{username}/{quantized_model_name}"
tokenizer = AutoTokenizer.from_pretrained(quantized_model_id)
model = GPTQModel.load(quantized_model_id, device="cuda:0") # or "cpu"
input_text = "This is a test prompt"
inputs = tokenizer(input_text, return_tensors="pt").to("cuda:0")
outputs = model.generate(**inputs)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))
package_section = """<details>
See pyproject.toml for the exact UV project file. See the GPTQModel repo for more details on how to install the package.
Use the provided pyproject.toml:
uv venv
source venv/bin/activate
uv sync
Average perplexity (PPL) on wikitext-2-raw-v1 dataset: 7.89 on on wikitext-2-raw-v1 dataset
This model is for research purposes only. It may inherit limitations and biases from the original model and the quantization process. Please use responsibly and refer to the original model card for more details.
For any questions or support, please visit ConfidentialMind or contact us directly.
license_section = f"""<details>
This model inherits the license from the original model. Please refer to the original model card for more details.
Original model card: {source_model}
This model was quantized by
Quantization performed using the GPTQModel pipeline and a big thanks to NeuralMagic for creating the calibration dataset, as well as the models original creators and/or fine-tuners.
TODO:
- HELMET
- Eluther evaluation harness
- Downloads last month
- 3
Inference Providers
NEW
This model is not currently available via any of the supported Inference Providers.
The model cannot be deployed to the HF Inference API:
The model has no library tag.