| base_model: meta-llama/Meta-Llama-3-8B | |
| dtype: bfloat16 | |
| merge_method: ties | |
| parameters: | |
| int8_mask: 1.0 | |
| normalize: 1.0 | |
| slices: | |
| - sources: | |
| - layer_range: [0, 32] | |
| model: meta-llama/Meta-Llama-3-8B | |
| - layer_range: [0, 32] | |
| model: meta-llama/Meta-Llama-3-8B-Instruct | |
| parameters: | |
| density: | |
| - filter: model.layers.0.self_attn.q_proj | |
| value: 1.0 | |
| - filter: model.layers.1.self_attn.q_proj | |
| value: 0.5915012245323329 | |
| - filter: model.layers.2.self_attn.q_proj | |
| value: 0.533635558334636 | |
| - filter: model.layers.3.self_attn.q_proj | |
| value: 0.5581262049919233 | |
| - filter: model.layers.4.self_attn.q_proj | |
| value: 0.5773018602469908 | |
| - filter: model.layers.5.self_attn.q_proj | |
| value: 0.5594288989630556 | |
| - filter: model.layers.6.self_attn.q_proj | |
| value: 0.5264707414934083 | |
| - filter: model.layers.7.self_attn.q_proj | |
| value: 0.5074253556354541 | |
| - filter: model.layers.8.self_attn.q_proj | |
| value: 0.5123234849669116 | |
| - filter: model.layers.9.self_attn.q_proj | |
| value: 0.549606586420718 | |
| - filter: model.layers.10.self_attn.q_proj | |
| value: 0.5104736595279037 | |
| - filter: model.layers.11.self_attn.q_proj | |
| value: 0.521937366473868 | |
| - filter: model.layers.12.self_attn.q_proj | |
| value: 0.5211036423323433 | |
| - filter: model.layers.13.self_attn.q_proj | |
| value: 0.5566932416236777 | |
| - filter: model.layers.14.self_attn.q_proj | |
| value: 0.5287113751237559 | |
| - filter: model.layers.15.self_attn.q_proj | |
| value: 0.5587254442186441 | |
| - filter: model.layers.16.self_attn.q_proj | |
| value: 0.5647699442446981 | |
| - filter: model.layers.17.self_attn.q_proj | |
| value: 0.5868636339951019 | |
| - filter: model.layers.18.self_attn.q_proj | |
| value: 0.590172476681778 | |
| - filter: model.layers.19.self_attn.q_proj | |
| value: 0.5772236986087228 | |
| - filter: model.layers.20.self_attn.q_proj | |
| value: 0.5837371684643844 | |
| - filter: model.layers.21.self_attn.q_proj | |
| value: 0.5988484185295191 | |
| - filter: model.layers.22.self_attn.q_proj | |
| value: 0.5956177374811109 | |
| - filter: model.layers.23.self_attn.q_proj | |
| value: 0.6109113647022042 | |
| - filter: model.layers.24.self_attn.q_proj | |
| value: 0.607680683653796 | |
| - filter: model.layers.25.self_attn.q_proj | |
| value: 0.612396435829295 | |
| - filter: model.layers.26.self_attn.q_proj | |
| value: 0.655307175238393 | |
| - filter: model.layers.27.self_attn.q_proj | |
| value: 0.6167734875722996 | |
| - filter: model.layers.28.self_attn.q_proj | |
| value: 0.6221405867333646 | |
| - filter: model.layers.29.self_attn.q_proj | |
| value: 0.5649783752800792 | |
| - filter: model.layers.30.self_attn.q_proj | |
| value: 0.5978323172320359 | |
| - filter: model.layers.31.self_attn.q_proj | |
| value: 0.5 | |
| - filter: model.layers.0.self_attn.k_proj | |
| value: 0.8480537752071284 | |
| - filter: model.layers.1.self_attn.k_proj | |
| value: 0.5569537804179042 | |
| - filter: model.layers.2.self_attn.k_proj | |
| value: 0.5319160022927414 | |
| - filter: model.layers.3.self_attn.k_proj | |
| value: 0.5251680475222761 | |
| - filter: model.layers.4.self_attn.k_proj | |
| value: 0.5493460476264915 | |
| - filter: model.layers.5.self_attn.k_proj | |
| value: 0.5465843364076911 | |
| - filter: model.layers.6.self_attn.k_proj | |
| value: 0.5242301078630608 | |
| - filter: model.layers.7.self_attn.k_proj | |
| value: 0.5145901724766818 | |
| - filter: model.layers.8.self_attn.k_proj | |
| value: 0.5039862435516649 | |
| - filter: model.layers.9.self_attn.k_proj | |
| value: 0.544526079933302 | |
| - filter: model.layers.10.self_attn.k_proj | |
| value: 0.5103694440102131 | |
| - filter: model.layers.11.self_attn.k_proj | |
| value: 0.5106560366838623 | |
| - filter: model.layers.12.self_attn.k_proj | |
| value: 0.5145641185972591 | |
| - filter: model.layers.13.self_attn.k_proj | |
| value: 0.5530717523839299 | |
| - filter: model.layers.14.self_attn.k_proj | |
| value: 0.5418685842321922 | |
| - filter: model.layers.15.self_attn.k_proj | |
| value: 0.5876713042572039 | |
| - filter: model.layers.16.self_attn.k_proj | |
| value: 0.6299046428013131 | |
| - filter: model.layers.17.self_attn.k_proj | |
| value: 0.6432963368245532 | |
| - filter: model.layers.18.self_attn.k_proj | |
| value: 0.6294877807305508 | |
| - filter: model.layers.19.self_attn.k_proj | |
| value: 0.604293679328852 | |
| - filter: model.layers.20.self_attn.k_proj | |
| value: 0.6142723151477255 | |
| - filter: model.layers.21.self_attn.k_proj | |
| value: 0.6172424574019072 | |
| - filter: model.layers.22.self_attn.k_proj | |
| value: 0.627559793653275 | |
| - filter: model.layers.23.self_attn.k_proj | |
| value: 0.6269345005471315 | |
| - filter: model.layers.24.self_attn.k_proj | |
| value: 0.6253973216611954 | |
| - filter: model.layers.25.self_attn.k_proj | |
| value: 0.6097128862487624 | |
| - filter: model.layers.26.self_attn.k_proj | |
| value: 0.654499504976291 | |
| - filter: model.layers.27.self_attn.k_proj | |
| value: 0.6062737741649732 | |
| - filter: model.layers.28.self_attn.k_proj | |
| value: 0.6051534573497994 | |
| - filter: model.layers.29.self_attn.k_proj | |
| value: 0.5576832890417384 | |
| - filter: model.layers.30.self_attn.k_proj | |
| value: 0.5832942525141994 | |
| - filter: model.layers.31.self_attn.k_proj | |
| value: 0.5001302693971132 | |
| - filter: model.layers.0.self_attn.v_proj | |
| value: 0.6780001042155177 | |
| - filter: model.layers.1.self_attn.v_proj | |
| value: 0.5999687353446929 | |
| - filter: model.layers.2.self_attn.v_proj | |
| value: 0.5700067740086499 | |
| - filter: model.layers.3.self_attn.v_proj | |
| value: 0.633447970402793 | |
| - filter: model.layers.4.self_attn.v_proj | |
| value: 0.6278203324475015 | |
| - filter: model.layers.5.self_attn.v_proj | |
| value: 0.5794382783596478 | |
| - filter: model.layers.6.self_attn.v_proj | |
| value: 0.5442655411390755 | |
| - filter: model.layers.7.self_attn.v_proj | |
| value: 0.5154499504976291 | |
| - filter: model.layers.8.self_attn.v_proj | |
| value: 0.5289719139179824 | |
| - filter: model.layers.9.self_attn.v_proj | |
| value: 0.522302120785785 | |
| - filter: model.layers.10.self_attn.v_proj | |
| value: 0.5297795841800844 | |
| - filter: model.layers.11.self_attn.v_proj | |
| value: 0.5429367932885206 | |
| - filter: model.layers.12.self_attn.v_proj | |
| value: 0.5740451253191601 | |
| - filter: model.layers.13.self_attn.v_proj | |
| value: 0.5515866812568392 | |
| - filter: model.layers.14.self_attn.v_proj | |
| value: 0.5319420561721641 | |
| - filter: model.layers.15.self_attn.v_proj | |
| value: 0.6064040435620864 | |
| - filter: model.layers.16.self_attn.v_proj | |
| value: 0.6014538064717837 | |
| - filter: model.layers.17.self_attn.v_proj | |
| value: 0.6136730759210046 | |
| - filter: model.layers.18.self_attn.v_proj | |
| value: 0.6215152936272211 | |
| - filter: model.layers.19.self_attn.v_proj | |
| value: 0.6304517742691886 | |
| - filter: model.layers.20.self_attn.v_proj | |
| value: 0.6145849617007972 | |
| - filter: model.layers.21.self_attn.v_proj | |
| value: 0.6035120629461727 | |
| - filter: model.layers.22.self_attn.v_proj | |
| value: 0.624537543640248 | |
| - filter: model.layers.23.self_attn.v_proj | |
| value: 0.6480641967588974 | |
| - filter: model.layers.24.self_attn.v_proj | |
| value: 0.6352456880829556 | |
| - filter: model.layers.25.self_attn.v_proj | |
| value: 0.5832681986347767 | |
| - filter: model.layers.26.self_attn.v_proj | |
| value: 0.6047887030378823 | |
| - filter: model.layers.27.self_attn.v_proj | |
| value: 0.5266010108905216 | |
| - filter: model.layers.28.self_attn.v_proj | |
| value: 0.5026835495805325 | |
| - filter: model.layers.29.self_attn.v_proj | |
| value: 0.5128966703142098 | |
| - filter: model.layers.30.self_attn.v_proj | |
| value: 0.559038090771716 | |
| - filter: model.layers.31.self_attn.v_proj | |
| value: 0.580011463706946 | |
| - filter: model.layers.0.self_attn.o_proj | |
| value: 0.5566150799854098 | |
| - filter: model.layers.1.self_attn.o_proj | |
| value: 0.516335782397999 | |
| - filter: model.layers.2.self_attn.o_proj | |
| value: 0.5357980303267157 | |
| - filter: model.layers.3.self_attn.o_proj | |
| value: 0.5400187587931843 | |
| - filter: model.layers.4.self_attn.o_proj | |
| value: 0.6263613151998333 | |
| - filter: model.layers.5.self_attn.o_proj | |
| value: 0.5280339742587671 | |
| - filter: model.layers.6.self_attn.o_proj | |
| value: 0.5044812672606951 | |
| - filter: model.layers.7.self_attn.o_proj | |
| value: 0.5143296336824553 | |
| - filter: model.layers.8.self_attn.o_proj | |
| value: 0.5011984784534417 | |
| - filter: model.layers.9.self_attn.o_proj | |
| value: 0.5095878276275337 | |
| - filter: model.layers.10.self_attn.o_proj | |
| value: 0.558595174821531 | |
| - filter: model.layers.11.self_attn.o_proj | |
| value: 0.5168047522276067 | |
| - filter: model.layers.12.self_attn.o_proj | |
| value: 0.586264394768381 | |
| - filter: model.layers.13.self_attn.o_proj | |
| value: 0.5044812672606951 | |
| - filter: model.layers.14.self_attn.o_proj | |
| value: 0.5224844979417436 | |
| - filter: model.layers.15.self_attn.o_proj | |
| value: 0.512584023761138 | |
| - filter: model.layers.16.self_attn.o_proj | |
| value: 0.5946016361836277 | |
| - filter: model.layers.17.self_attn.o_proj | |
| value: 0.5662289614923662 | |
| - filter: model.layers.18.self_attn.o_proj | |
| value: 0.5052368297639519 | |
| - filter: model.layers.19.self_attn.o_proj | |
| value: 0.5294669376270127 | |
| - filter: model.layers.20.self_attn.o_proj | |
| value: 0.6374602678338804 | |
| - filter: model.layers.21.self_attn.o_proj | |
| value: 0.583763222343807 | |
| - filter: model.layers.22.self_attn.o_proj | |
| value: 0.694752748684279 | |
| - filter: model.layers.23.self_attn.o_proj | |
| value: 0.5834505757907352 | |
| - filter: model.layers.24.self_attn.o_proj | |
| value: 0.57493095721953 | |
| - filter: model.layers.25.self_attn.o_proj | |
| value: 0.9771507477463395 | |
| - filter: model.layers.26.self_attn.o_proj | |
| value: 0.5683132718461779 | |
| - filter: model.layers.27.self_attn.o_proj | |
| value: 0.8806992861237039 | |
| - filter: model.layers.28.self_attn.o_proj | |
| value: 0.585013808556094 | |
| - filter: model.layers.29.self_attn.o_proj | |
| value: 0.7172111927466 | |
| - filter: model.layers.30.self_attn.o_proj | |
| value: 0.539158980772237 | |
| - filter: model.layers.31.self_attn.o_proj | |
| value: 0.5273565733937783 | |
| - filter: model.layers.0.mlp.gate_proj | |
| value: 0.7398259600854568 | |
| - filter: model.layers.1.mlp.gate_proj | |
| value: 0.6623156688030848 | |
| - filter: model.layers.2.mlp.gate_proj | |
| value: 0.6218800479391382 | |
| - filter: model.layers.3.mlp.gate_proj | |
| value: 0.6496795372831015 | |
| - filter: model.layers.4.mlp.gate_proj | |
| value: 0.6073419832213016 | |
| - filter: model.layers.5.mlp.gate_proj | |
| value: 0.5901203689229326 | |
| - filter: model.layers.6.mlp.gate_proj | |
| value: 0.5831639831170862 | |
| - filter: model.layers.7.mlp.gate_proj | |
| value: 0.5327757803136887 | |
| - filter: model.layers.8.mlp.gate_proj | |
| value: 0.5048460215726122 | |
| - filter: model.layers.9.mlp.gate_proj | |
| value: 0.517299775936637 | |
| - filter: model.layers.10.mlp.gate_proj | |
| value: 0.5256109634724611 | |
| - filter: model.layers.11.mlp.gate_proj | |
| value: 0.5363191079151686 | |
| - filter: model.layers.12.mlp.gate_proj | |
| value: 0.5258454483872649 | |
| - filter: model.layers.13.mlp.gate_proj | |
| value: 0.5099004741806055 | |
| - filter: model.layers.14.mlp.gate_proj | |
| value: 0.544161325621385 | |
| - filter: model.layers.15.mlp.gate_proj | |
| value: 0.5567974571413683 | |
| - filter: model.layers.16.mlp.gate_proj | |
| value: 0.5865249335626075 | |
| - filter: model.layers.17.mlp.gate_proj | |
| value: 0.6023917461309989 | |
| - filter: model.layers.18.mlp.gate_proj | |
| value: 0.6177114272315147 | |
| - filter: model.layers.19.mlp.gate_proj | |
| value: 0.614011776353499 | |
| - filter: model.layers.20.mlp.gate_proj | |
| value: 0.6049189724349956 | |
| - filter: model.layers.21.mlp.gate_proj | |
| value: 0.5991871189620135 | |
| - filter: model.layers.22.mlp.gate_proj | |
| value: 0.6197957375853265 | |
| - filter: model.layers.23.mlp.gate_proj | |
| value: 0.6167734875722996 | |
| - filter: model.layers.24.mlp.gate_proj | |
| value: 0.5977281017143453 | |
| - filter: model.layers.25.mlp.gate_proj | |
| value: 0.6520764941899849 | |
| - filter: model.layers.26.mlp.gate_proj | |
| value: 0.5683653796050232 | |
| - filter: model.layers.27.mlp.gate_proj | |
| value: 0.5564587567088739 | |
| - filter: model.layers.28.mlp.gate_proj | |
| value: 0.5045333750195404 | |
| - filter: model.layers.29.mlp.gate_proj | |
| value: 0.5245427544161325 | |
| - filter: model.layers.30.mlp.gate_proj | |
| value: 0.5658902610598718 | |
| - filter: model.layers.31.mlp.gate_proj | |
| value: 0.5598978687926632 | |
| - filter: model.layers.0.mlp.up_proj | |
| value: 0.7446198738992236 | |
| - filter: model.layers.1.mlp.up_proj | |
| value: 0.6871971236517117 | |
| - filter: model.layers.2.mlp.up_proj | |
| value: 0.6360012505862123 | |
| - filter: model.layers.3.mlp.up_proj | |
| value: 0.6525194101401699 | |
| - filter: model.layers.4.mlp.up_proj | |
| value: 0.5973112396435829 | |
| - filter: model.layers.5.mlp.up_proj | |
| value: 0.5818873430253765 | |
| - filter: model.layers.6.mlp.up_proj | |
| value: 0.5714397373768955 | |
| - filter: model.layers.7.mlp.up_proj | |
| value: 0.5230576832890418 | |
| - filter: model.layers.8.mlp.up_proj | |
| value: 0.5084935646917826 | |
| - filter: model.layers.9.mlp.up_proj | |
| value: 0.5306393622010317 | |
| - filter: model.layers.10.mlp.up_proj | |
| value: 0.5488249700380387 | |
| - filter: model.layers.11.mlp.up_proj | |
| value: 0.5646657287270075 | |
| - filter: model.layers.12.mlp.up_proj | |
| value: 0.557266426970976 | |
| - filter: model.layers.13.mlp.up_proj | |
| value: 0.5164399979156896 | |
| - filter: model.layers.14.mlp.up_proj | |
| value: 0.5224584440623209 | |
| - filter: model.layers.15.mlp.up_proj | |
| value: 0.5427283622531395 | |
| - filter: model.layers.16.mlp.up_proj | |
| value: 0.5844927309676411 | |
| - filter: model.layers.17.mlp.up_proj | |
| value: 0.5983012870616435 | |
| - filter: model.layers.18.mlp.up_proj | |
| value: 0.6132301599708196 | |
| - filter: model.layers.19.mlp.up_proj | |
| value: 0.6065603668386222 | |
| - filter: model.layers.20.mlp.up_proj | |
| value: 0.598040748267417 | |
| - filter: model.layers.21.mlp.up_proj | |
| value: 0.5880360585691209 | |
| - filter: model.layers.22.mlp.up_proj | |
| value: 0.6064561513209317 | |
| - filter: model.layers.23.mlp.up_proj | |
| value: 0.6022093689750404 | |
| - filter: model.layers.24.mlp.up_proj | |
| value: 0.580376218018863 | |
| - filter: model.layers.25.mlp.up_proj | |
| value: 0.6365223281746653 | |
| - filter: model.layers.26.mlp.up_proj | |
| value: 0.5579698817153874 | |
| - filter: model.layers.27.mlp.up_proj | |
| value: 0.5470272523578761 | |
| - filter: model.layers.28.mlp.up_proj | |
| value: 0.5102131207336772 | |
| - filter: model.layers.29.mlp.up_proj | |
| value: 0.5205044031056224 | |
| - filter: model.layers.30.mlp.up_proj | |
| value: 0.555651086446772 | |
| - filter: model.layers.31.mlp.up_proj | |
| value: 0.6076025220155281 | |
| - filter: model.layers.0.mlp.down_proj | |
| value: 0.5857954249387733 | |
| - filter: model.layers.1.mlp.down_proj | |
| value: 0.5853264551091657 | |
| - filter: model.layers.2.mlp.down_proj | |
| value: 0.5342087436819343 | |
| - filter: model.layers.3.mlp.down_proj | |
| value: 0.5606534312959199 | |
| - filter: model.layers.4.mlp.down_proj | |
| value: 0.566541608045438 | |
| - filter: model.layers.5.mlp.down_proj | |
| value: 0.5318899484133187 | |
| - filter: model.layers.6.mlp.down_proj | |
| value: 0.538351310510135 | |
| - filter: model.layers.7.mlp.down_proj | |
| value: 0.5209994268146527 | |
| - filter: model.layers.8.mlp.down_proj | |
| value: 0.5305351466833411 | |
| - filter: model.layers.9.mlp.down_proj | |
| value: 0.503673596998593 | |
| - filter: model.layers.10.mlp.down_proj | |
| value: 0.5296232609035485 | |
| - filter: model.layers.11.mlp.down_proj | |
| value: 0.5282163514147257 | |
| - filter: model.layers.12.mlp.down_proj | |
| value: 0.549606586420718 | |
| - filter: model.layers.13.mlp.down_proj | |
| value: 0.5241519462247929 | |
| - filter: model.layers.14.mlp.down_proj | |
| value: 0.5465582825282684 | |
| - filter: model.layers.15.mlp.down_proj | |
| value: 0.5839977072586108 | |
| - filter: model.layers.16.mlp.down_proj | |
| value: 0.5891824292637173 | |
| - filter: model.layers.17.mlp.down_proj | |
| value: 0.570215205044031 | |
| - filter: model.layers.18.mlp.down_proj | |
| value: 0.5918920327236725 | |
| - filter: model.layers.19.mlp.down_proj | |
| value: 0.6033557396696368 | |
| - filter: model.layers.20.mlp.down_proj | |
| value: 0.5831900369965088 | |
| - filter: model.layers.21.mlp.down_proj | |
| value: 0.6060392892501694 | |
| - filter: model.layers.22.mlp.down_proj | |
| value: 0.5592465218070971 | |
| - filter: model.layers.23.mlp.down_proj | |
| value: 0.5628159032879996 | |
| - filter: model.layers.24.mlp.down_proj | |
| value: 0.5611484550049503 | |
| - filter: model.layers.25.mlp.down_proj | |
| value: 0.6383460997342505 | |
| - filter: model.layers.26.mlp.down_proj | |
| value: 0.5379865561982179 | |
| - filter: model.layers.27.mlp.down_proj | |
| value: 0.547183575634412 | |
| - filter: model.layers.28.mlp.down_proj | |
| value: 0.5039862435516649 | |
| - filter: model.layers.29.mlp.down_proj | |
| value: 0.8178312750768589 | |
| - filter: model.layers.30.mlp.down_proj | |
| value: 0.7657756239904121 | |
| - filter: model.layers.31.mlp.down_proj | |
| value: 0.8403939346568705 | |
| - value: 1.0 | |
| weight: | |
| - value: 1.0 | |
| tokenizer_source: union |