Hugging Face's logo

GreenBitAI
/
Llama-2-7B-channel-mix-bpw-2.5

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
Model card Files Community
Llama-2-7B-channel-mix-bpw-2.5 / quant_strategy.json
NicoNico6
update
8ca1194
raw
history blame contribute delete
85.8 kB
{
"measurement": {
"model.layers.0": {
"accuracy": 0.9382696151733398,
"total_bits": 446084128,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
"model.layers.1": {
"accuracy": 0.9415526390075684,
"total_bits": 578335776,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
},
"model.layers.2": {
"accuracy": 0.9650976061820984,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.3": {
"accuracy": 0.9562180042266846,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.4": {
"accuracy": 0.9498416185379028,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.5": {
"accuracy": 0.9452450275421143,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.6": {
"accuracy": 0.9379130601882935,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.7": {
"accuracy": 0.932453989982605,
"total_bits": 424064032,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.01,
0.99
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
}
},
"model.layers.8": {
"accuracy": 0.9342072010040283,
"total_bits": 446084128,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
"model.layers.9": {
"accuracy": 0.932488203048706,
"total_bits": 446084128,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
"model.layers.10": {
"accuracy": 0.9288740158081055,
"total_bits": 446084128,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.02,
0.98
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
"model.layers.11": {
"accuracy": 0.9333758354187012,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.12": {
"accuracy": 0.9322373867034912,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.13": {
"accuracy": 0.9323720932006836,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.14": {
"accuracy": 0.9289882183074951,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.15": {
"accuracy": 0.9291183948516846,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.16": {
"accuracy": 0.9419481754302979,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.17": {
"accuracy": 0.9265720844268799,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.18": {
"accuracy": 0.9266581535339355,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.19": {
"accuracy": 0.9268605709075928,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.20": {
"accuracy": 0.9419999122619629,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.21": {
"accuracy": 0.928084135055542,
"total_bits": 478983200,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.22": {
"accuracy": 0.941460132598877,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.23": {
"accuracy": 0.9415285587310791,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.24": {
"accuracy": 0.9416172504425049,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.25": {
"accuracy": 0.9402132034301758,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.26": {
"accuracy": 0.939831018447876,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.27": {
"accuracy": 0.9388439655303955,
"total_bits": 569947168,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.28": {
"accuracy": 0.9380154609680176,
"total_bits": 578335776,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
},
"model.layers.29": {
"accuracy": 0.9371981620788574,
"total_bits": 578335776,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
},
"model.layers.30": {
"accuracy": 0.93943190574646,
"total_bits": 675329056,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.31": {
"accuracy": 0.9260892868041992,
"total_bits": 578335776,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
}
}
}