Hugging Face's logo

GreenBitAI
/
Llama-2-13B-channel-mix-bpw-3.0

Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
Model card Files Community
Llama-2-13B-channel-mix-bpw-3.0 / quant_strategy.json
NicoNico6
update
0e62d10
raw
history blame contribute delete
104 kB
{
"measurement": {
"model.layers.0": {
"accuracy": 0.9380207061767578,
"total_bits": 727679296,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.2,
0.8
],
"scale_bits": 4
}
},
"model.layers.1": {
"accuracy": 0.9466156959533691,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.2": {
"accuracy": 0.9514789581298828,
"total_bits": 1168605504,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.3": {
"accuracy": 0.9370343685150146,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.4": {
"accuracy": 0.9433646202087402,
"total_bits": 744980800,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.5": {
"accuracy": 0.9492185115814209,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.6": {
"accuracy": 0.9399415254592896,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.7": {
"accuracy": 0.9425477981567383,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.8": {
"accuracy": 0.9373887777328491,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.9": {
"accuracy": 0.9468040466308594,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.10": {
"accuracy": 0.943734884262085,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.11": {
"accuracy": 0.9423996210098267,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.12": {
"accuracy": 0.9403209686279297,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.13": {
"accuracy": 0.9378421306610107,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.14": {
"accuracy": 0.9530701637268066,
"total_bits": 1168605504,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.15": {
"accuracy": 0.9530957937240601,
"total_bits": 1168605504,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.16": {
"accuracy": 0.9536402225494385,
"total_bits": 1168605504,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.75,
0.25
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.17": {
"accuracy": 0.9361867904663086,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.18": {
"accuracy": 0.939406156539917,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.19": {
"accuracy": 0.9403789043426514,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.20": {
"accuracy": 0.9423317909240723,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.21": {
"accuracy": 0.9440984725952148,
"total_bits": 1058505024,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.22": {
"accuracy": 0.9362955093383789,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.23": {
"accuracy": 0.9399640560150146,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.24": {
"accuracy": 0.9426877498626709,
"total_bits": 948404544,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.25,
0.75
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1
],
"scale_bits": 4
}
},
"model.layers.25": {
"accuracy": 0.9379689693450928,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.26": {
"accuracy": 0.9405760765075684,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.27": {
"accuracy": 0.9437229633331299,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.28": {
"accuracy": 0.945061445236206,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.29": {
"accuracy": 0.9464197158813477,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.30": {
"accuracy": 0.9468019008636475,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.31": {
"accuracy": 0.947880744934082,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.32": {
"accuracy": 0.9481966495513916,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.33": {
"accuracy": 0.9368886947631836,
"total_bits": 744980800,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
},
"model.layers.34": {
"accuracy": 0.947458028793335,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.35": {
"accuracy": 0.9465105533599854,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.36": {
"accuracy": 0.9439797401428223,
"total_bits": 882344256,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.1,
0.9
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128
},
"bits": [
4
],
"bits_prop": [
1.0
],
"scale_bits": 4
}
},
"model.layers.37": {
"accuracy": 0.9462072849273682,
"total_bits": 895451456,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
},
"model.layers.38": {
"accuracy": 0.9470703601837158,
"total_bits": 895451456,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.35,
0.65
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.5,
0.5
],
"scale_bits": 4
}
},
"model.layers.39": {
"accuracy": 0.9403278827667236,
"total_bits": 744980800,
"q_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"k_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"v_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
},
"o_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"up_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"gate_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.05,
0.95
],
"scale_bits": 4
},
"down_proj": {
"group_size": {
"4": 128,
"2": 128
},
"bits": [
4,
2
],
"bits_prop": [
0.4,
0.6
],
"scale_bits": 4
}
}
}
}