Hugging Face's logo

GreenBitAI
/
Llama-2-13B-Chat-layer-mix-bpw-2.5

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
Model card Files Community
Llama-2-13B-Chat-layer-mix-bpw-2.5 / quant_strategy.json
NicoNico6
update
f9e6685
raw
history blame contribute delete
86.1 kB
{
"measurement": {
"model.layers.0": {
"accuracy": 0.7753477096557617,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.1": {
"accuracy": 0.825984001159668,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.2": {
"accuracy": 0.8247356414794922,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.3": {
"accuracy": 0.7183780670166016,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.4": {
"accuracy": 0.9247467517852783,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.5": {
"accuracy": 0.9151527881622314,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.6": {
"accuracy": 0.9062843322753906,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.7": {
"accuracy": 0.8995904922485352,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.8": {
"accuracy": 0.8948485851287842,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.9": {
"accuracy": 0.8846879005432129,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.10": {
"accuracy": 0.8857412338256836,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.11": {
"accuracy": 0.8858399391174316,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.12": {
"accuracy": 0.8839669227600098,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.13": {
"accuracy": 0.88348388671875,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.14": {
"accuracy": 0.8837137222290039,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.15": {
"accuracy": 0.8834319114685059,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.16": {
"accuracy": 0.8816227912902832,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.17": {
"accuracy": 0.8801741600036621,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.18": {
"accuracy": 0.8858942985534668,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.19": {
"accuracy": 0.8844828605651855,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.20": {
"accuracy": 0.884763240814209,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.21": {
"accuracy": 0.8864822387695312,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.22": {
"accuracy": 0.8917703628540039,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.23": {
"accuracy": 0.8959054946899414,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.24": {
"accuracy": 0.9000020027160645,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.25": {
"accuracy": 0.9012517929077148,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.26": {
"accuracy": 0.9033737182617188,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.27": {
"accuracy": 0.9054808616638184,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.28": {
"accuracy": 0.9051413536071777,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.29": {
"accuracy": 0.9067106246948242,
"total_bits": 705854208,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.30": {
"accuracy": 0.9287574291229248,
"total_bits": 845194944,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.31": {
"accuracy": 0.9299304485321045,
"total_bits": 845194944,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.32": {
"accuracy": 0.9279956817626953,
"total_bits": 845194944,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.33": {
"accuracy": 0.9499871730804443,
"total_bits": 1036146624,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.34": {
"accuracy": 0.9488976001739502,
"total_bits": 1036146624,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.35": {
"accuracy": 0.9470212459564209,
"total_bits": 1036146624,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.36": {
"accuracy": 0.9721497297286987,
"total_bits": 1175490624,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.37": {
"accuracy": 0.9775956869125366,
"total_bits": 1278705984,
"q_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.38": {
"accuracy": 0.971349835395813,
"total_bits": 1227098304,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.39": {
"accuracy": 0.9591939449310303,
"total_bits": 1175490624,
"q_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"2": 64
},
"bits": [
2
],
"bits_prop": [
1
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
}
}
}