--- base_model: - meta-llama/Meta-Llama-3-8B - meta-llama/Meta-Llama-3-8B-Instruct library_name: transformers tags: - mergekit - merge --- # Untitled Model (1) This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit). ## Merge Details ### Merge Method This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) as a base. ### Models Merged The following models were included in the merge: * [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) ### Configuration The following YAML configuration was used to produce this model: ```yaml base_model: meta-llama/Meta-Llama-3-8B dtype: bfloat16 merge_method: ties parameters: int8_mask: 1.0 normalize: 1.0 slices: - sources: - layer_range: [0, 32] model: meta-llama/Meta-Llama-3-8B - layer_range: [0, 32] model: meta-llama/Meta-Llama-3-8B-Instruct parameters: density: - filter: model.layers.0.self_attn.q_proj value: 1.0 - filter: model.layers.1.self_attn.q_proj value: 0.5915012245323329 - filter: model.layers.2.self_attn.q_proj value: 0.533635558334636 - filter: model.layers.3.self_attn.q_proj value: 0.5581262049919233 - filter: model.layers.4.self_attn.q_proj value: 0.5773018602469908 - filter: model.layers.5.self_attn.q_proj value: 0.5594288989630556 - filter: model.layers.6.self_attn.q_proj value: 0.5264707414934083 - filter: model.layers.7.self_attn.q_proj value: 0.5074253556354541 - filter: model.layers.8.self_attn.q_proj value: 0.5123234849669116 - filter: model.layers.9.self_attn.q_proj value: 0.549606586420718 - filter: model.layers.10.self_attn.q_proj value: 0.5104736595279037 - filter: model.layers.11.self_attn.q_proj value: 0.521937366473868 - filter: model.layers.12.self_attn.q_proj value: 0.5211036423323433 - filter: model.layers.13.self_attn.q_proj value: 0.5566932416236777 - filter: model.layers.14.self_attn.q_proj value: 0.5287113751237559 - filter: model.layers.15.self_attn.q_proj value: 0.5587254442186441 - filter: model.layers.16.self_attn.q_proj value: 0.5647699442446981 - filter: model.layers.17.self_attn.q_proj value: 0.5868636339951019 - filter: model.layers.18.self_attn.q_proj value: 0.590172476681778 - filter: model.layers.19.self_attn.q_proj value: 0.5772236986087228 - filter: model.layers.20.self_attn.q_proj value: 0.5837371684643844 - filter: model.layers.21.self_attn.q_proj value: 0.5988484185295191 - filter: model.layers.22.self_attn.q_proj value: 0.5956177374811109 - filter: model.layers.23.self_attn.q_proj value: 0.6109113647022042 - filter: model.layers.24.self_attn.q_proj value: 0.607680683653796 - filter: model.layers.25.self_attn.q_proj value: 0.612396435829295 - filter: model.layers.26.self_attn.q_proj value: 0.655307175238393 - filter: model.layers.27.self_attn.q_proj value: 0.6167734875722996 - filter: model.layers.28.self_attn.q_proj value: 0.6221405867333646 - filter: model.layers.29.self_attn.q_proj value: 0.5649783752800792 - filter: model.layers.30.self_attn.q_proj value: 0.5978323172320359 - filter: model.layers.31.self_attn.q_proj value: 0.5 - filter: model.layers.0.self_attn.k_proj value: 0.8480537752071284 - filter: model.layers.1.self_attn.k_proj value: 0.5569537804179042 - filter: model.layers.2.self_attn.k_proj value: 0.5319160022927414 - filter: model.layers.3.self_attn.k_proj value: 0.5251680475222761 - filter: model.layers.4.self_attn.k_proj value: 0.5493460476264915 - filter: model.layers.5.self_attn.k_proj value: 0.5465843364076911 - filter: model.layers.6.self_attn.k_proj value: 0.5242301078630608 - filter: model.layers.7.self_attn.k_proj value: 0.5145901724766818 - filter: model.layers.8.self_attn.k_proj value: 0.5039862435516649 - filter: model.layers.9.self_attn.k_proj value: 0.544526079933302 - filter: model.layers.10.self_attn.k_proj value: 0.5103694440102131 - filter: model.layers.11.self_attn.k_proj value: 0.5106560366838623 - filter: model.layers.12.self_attn.k_proj value: 0.5145641185972591 - filter: model.layers.13.self_attn.k_proj value: 0.5530717523839299 - filter: model.layers.14.self_attn.k_proj value: 0.5418685842321922 - filter: model.layers.15.self_attn.k_proj value: 0.5876713042572039 - filter: model.layers.16.self_attn.k_proj value: 0.6299046428013131 - filter: model.layers.17.self_attn.k_proj value: 0.6432963368245532 - filter: model.layers.18.self_attn.k_proj value: 0.6294877807305508 - filter: model.layers.19.self_attn.k_proj value: 0.604293679328852 - filter: model.layers.20.self_attn.k_proj value: 0.6142723151477255 - filter: model.layers.21.self_attn.k_proj value: 0.6172424574019072 - filter: model.layers.22.self_attn.k_proj value: 0.627559793653275 - filter: model.layers.23.self_attn.k_proj value: 0.6269345005471315 - filter: model.layers.24.self_attn.k_proj value: 0.6253973216611954 - filter: model.layers.25.self_attn.k_proj value: 0.6097128862487624 - filter: model.layers.26.self_attn.k_proj value: 0.654499504976291 - filter: model.layers.27.self_attn.k_proj value: 0.6062737741649732 - filter: model.layers.28.self_attn.k_proj value: 0.6051534573497994 - filter: model.layers.29.self_attn.k_proj value: 0.5576832890417384 - filter: model.layers.30.self_attn.k_proj value: 0.5832942525141994 - filter: model.layers.31.self_attn.k_proj value: 0.5001302693971132 - filter: model.layers.0.self_attn.v_proj value: 0.6780001042155177 - filter: model.layers.1.self_attn.v_proj value: 0.5999687353446929 - filter: model.layers.2.self_attn.v_proj value: 0.5700067740086499 - filter: model.layers.3.self_attn.v_proj value: 0.633447970402793 - filter: model.layers.4.self_attn.v_proj value: 0.6278203324475015 - filter: model.layers.5.self_attn.v_proj value: 0.5794382783596478 - filter: model.layers.6.self_attn.v_proj value: 0.5442655411390755 - filter: model.layers.7.self_attn.v_proj value: 0.5154499504976291 - filter: model.layers.8.self_attn.v_proj value: 0.5289719139179824 - filter: model.layers.9.self_attn.v_proj value: 0.522302120785785 - filter: model.layers.10.self_attn.v_proj value: 0.5297795841800844 - filter: model.layers.11.self_attn.v_proj value: 0.5429367932885206 - filter: model.layers.12.self_attn.v_proj value: 0.5740451253191601 - filter: model.layers.13.self_attn.v_proj value: 0.5515866812568392 - filter: model.layers.14.self_attn.v_proj value: 0.5319420561721641 - filter: model.layers.15.self_attn.v_proj value: 0.6064040435620864 - filter: model.layers.16.self_attn.v_proj value: 0.6014538064717837 - filter: model.layers.17.self_attn.v_proj value: 0.6136730759210046 - filter: model.layers.18.self_attn.v_proj value: 0.6215152936272211 - filter: model.layers.19.self_attn.v_proj value: 0.6304517742691886 - filter: model.layers.20.self_attn.v_proj value: 0.6145849617007972 - filter: model.layers.21.self_attn.v_proj value: 0.6035120629461727 - filter: model.layers.22.self_attn.v_proj value: 0.624537543640248 - filter: model.layers.23.self_attn.v_proj value: 0.6480641967588974 - filter: model.layers.24.self_attn.v_proj value: 0.6352456880829556 - filter: model.layers.25.self_attn.v_proj value: 0.5832681986347767 - filter: model.layers.26.self_attn.v_proj value: 0.6047887030378823 - filter: model.layers.27.self_attn.v_proj value: 0.5266010108905216 - filter: model.layers.28.self_attn.v_proj value: 0.5026835495805325 - filter: model.layers.29.self_attn.v_proj value: 0.5128966703142098 - filter: model.layers.30.self_attn.v_proj value: 0.559038090771716 - filter: model.layers.31.self_attn.v_proj value: 0.580011463706946 - filter: model.layers.0.self_attn.o_proj value: 0.5566150799854098 - filter: model.layers.1.self_attn.o_proj value: 0.516335782397999 - filter: model.layers.2.self_attn.o_proj value: 0.5357980303267157 - filter: model.layers.3.self_attn.o_proj value: 0.5400187587931843 - filter: model.layers.4.self_attn.o_proj value: 0.6263613151998333 - filter: model.layers.5.self_attn.o_proj value: 0.5280339742587671 - filter: model.layers.6.self_attn.o_proj value: 0.5044812672606951 - filter: model.layers.7.self_attn.o_proj value: 0.5143296336824553 - filter: model.layers.8.self_attn.o_proj value: 0.5011984784534417 - filter: model.layers.9.self_attn.o_proj value: 0.5095878276275337 - filter: model.layers.10.self_attn.o_proj value: 0.558595174821531 - filter: model.layers.11.self_attn.o_proj value: 0.5168047522276067 - filter: model.layers.12.self_attn.o_proj value: 0.586264394768381 - filter: model.layers.13.self_attn.o_proj value: 0.5044812672606951 - filter: model.layers.14.self_attn.o_proj value: 0.5224844979417436 - filter: model.layers.15.self_attn.o_proj value: 0.512584023761138 - filter: model.layers.16.self_attn.o_proj value: 0.5946016361836277 - filter: model.layers.17.self_attn.o_proj value: 0.5662289614923662 - filter: model.layers.18.self_attn.o_proj value: 0.5052368297639519 - filter: model.layers.19.self_attn.o_proj value: 0.5294669376270127 - filter: model.layers.20.self_attn.o_proj value: 0.6374602678338804 - filter: model.layers.21.self_attn.o_proj value: 0.583763222343807 - filter: model.layers.22.self_attn.o_proj value: 0.694752748684279 - filter: model.layers.23.self_attn.o_proj value: 0.5834505757907352 - filter: model.layers.24.self_attn.o_proj value: 0.57493095721953 - filter: model.layers.25.self_attn.o_proj value: 0.9771507477463395 - filter: model.layers.26.self_attn.o_proj value: 0.5683132718461779 - filter: model.layers.27.self_attn.o_proj value: 0.8806992861237039 - filter: model.layers.28.self_attn.o_proj value: 0.585013808556094 - filter: model.layers.29.self_attn.o_proj value: 0.7172111927466 - filter: model.layers.30.self_attn.o_proj value: 0.539158980772237 - filter: model.layers.31.self_attn.o_proj value: 0.5273565733937783 - filter: model.layers.0.mlp.gate_proj value: 0.7398259600854568 - filter: model.layers.1.mlp.gate_proj value: 0.6623156688030848 - filter: model.layers.2.mlp.gate_proj value: 0.6218800479391382 - filter: model.layers.3.mlp.gate_proj value: 0.6496795372831015 - filter: model.layers.4.mlp.gate_proj value: 0.6073419832213016 - filter: model.layers.5.mlp.gate_proj value: 0.5901203689229326 - filter: model.layers.6.mlp.gate_proj value: 0.5831639831170862 - filter: model.layers.7.mlp.gate_proj value: 0.5327757803136887 - filter: model.layers.8.mlp.gate_proj value: 0.5048460215726122 - filter: model.layers.9.mlp.gate_proj value: 0.517299775936637 - filter: model.layers.10.mlp.gate_proj value: 0.5256109634724611 - filter: model.layers.11.mlp.gate_proj value: 0.5363191079151686 - filter: model.layers.12.mlp.gate_proj value: 0.5258454483872649 - filter: model.layers.13.mlp.gate_proj value: 0.5099004741806055 - filter: model.layers.14.mlp.gate_proj value: 0.544161325621385 - filter: model.layers.15.mlp.gate_proj value: 0.5567974571413683 - filter: model.layers.16.mlp.gate_proj value: 0.5865249335626075 - filter: model.layers.17.mlp.gate_proj value: 0.6023917461309989 - filter: model.layers.18.mlp.gate_proj value: 0.6177114272315147 - filter: model.layers.19.mlp.gate_proj value: 0.614011776353499 - filter: model.layers.20.mlp.gate_proj value: 0.6049189724349956 - filter: model.layers.21.mlp.gate_proj value: 0.5991871189620135 - filter: model.layers.22.mlp.gate_proj value: 0.6197957375853265 - filter: model.layers.23.mlp.gate_proj value: 0.6167734875722996 - filter: model.layers.24.mlp.gate_proj value: 0.5977281017143453 - filter: model.layers.25.mlp.gate_proj value: 0.6520764941899849 - filter: model.layers.26.mlp.gate_proj value: 0.5683653796050232 - filter: model.layers.27.mlp.gate_proj value: 0.5564587567088739 - filter: model.layers.28.mlp.gate_proj value: 0.5045333750195404 - filter: model.layers.29.mlp.gate_proj value: 0.5245427544161325 - filter: model.layers.30.mlp.gate_proj value: 0.5658902610598718 - filter: model.layers.31.mlp.gate_proj value: 0.5598978687926632 - filter: model.layers.0.mlp.up_proj value: 0.7446198738992236 - filter: model.layers.1.mlp.up_proj value: 0.6871971236517117 - filter: model.layers.2.mlp.up_proj value: 0.6360012505862123 - filter: model.layers.3.mlp.up_proj value: 0.6525194101401699 - filter: model.layers.4.mlp.up_proj value: 0.5973112396435829 - filter: model.layers.5.mlp.up_proj value: 0.5818873430253765 - filter: model.layers.6.mlp.up_proj value: 0.5714397373768955 - filter: model.layers.7.mlp.up_proj value: 0.5230576832890418 - filter: model.layers.8.mlp.up_proj value: 0.5084935646917826 - filter: model.layers.9.mlp.up_proj value: 0.5306393622010317 - filter: model.layers.10.mlp.up_proj value: 0.5488249700380387 - filter: model.layers.11.mlp.up_proj value: 0.5646657287270075 - filter: model.layers.12.mlp.up_proj value: 0.557266426970976 - filter: model.layers.13.mlp.up_proj value: 0.5164399979156896 - filter: model.layers.14.mlp.up_proj value: 0.5224584440623209 - filter: model.layers.15.mlp.up_proj value: 0.5427283622531395 - filter: model.layers.16.mlp.up_proj value: 0.5844927309676411 - filter: model.layers.17.mlp.up_proj value: 0.5983012870616435 - filter: model.layers.18.mlp.up_proj value: 0.6132301599708196 - filter: model.layers.19.mlp.up_proj value: 0.6065603668386222 - filter: model.layers.20.mlp.up_proj value: 0.598040748267417 - filter: model.layers.21.mlp.up_proj value: 0.5880360585691209 - filter: model.layers.22.mlp.up_proj value: 0.6064561513209317 - filter: model.layers.23.mlp.up_proj value: 0.6022093689750404 - filter: model.layers.24.mlp.up_proj value: 0.580376218018863 - filter: model.layers.25.mlp.up_proj value: 0.6365223281746653 - filter: model.layers.26.mlp.up_proj value: 0.5579698817153874 - filter: model.layers.27.mlp.up_proj value: 0.5470272523578761 - filter: model.layers.28.mlp.up_proj value: 0.5102131207336772 - filter: model.layers.29.mlp.up_proj value: 0.5205044031056224 - filter: model.layers.30.mlp.up_proj value: 0.555651086446772 - filter: model.layers.31.mlp.up_proj value: 0.6076025220155281 - filter: model.layers.0.mlp.down_proj value: 0.5857954249387733 - filter: model.layers.1.mlp.down_proj value: 0.5853264551091657 - filter: model.layers.2.mlp.down_proj value: 0.5342087436819343 - filter: model.layers.3.mlp.down_proj value: 0.5606534312959199 - filter: model.layers.4.mlp.down_proj value: 0.566541608045438 - filter: model.layers.5.mlp.down_proj value: 0.5318899484133187 - filter: model.layers.6.mlp.down_proj value: 0.538351310510135 - filter: model.layers.7.mlp.down_proj value: 0.5209994268146527 - filter: model.layers.8.mlp.down_proj value: 0.5305351466833411 - filter: model.layers.9.mlp.down_proj value: 0.503673596998593 - filter: model.layers.10.mlp.down_proj value: 0.5296232609035485 - filter: model.layers.11.mlp.down_proj value: 0.5282163514147257 - filter: model.layers.12.mlp.down_proj value: 0.549606586420718 - filter: model.layers.13.mlp.down_proj value: 0.5241519462247929 - filter: model.layers.14.mlp.down_proj value: 0.5465582825282684 - filter: model.layers.15.mlp.down_proj value: 0.5839977072586108 - filter: model.layers.16.mlp.down_proj value: 0.5891824292637173 - filter: model.layers.17.mlp.down_proj value: 0.570215205044031 - filter: model.layers.18.mlp.down_proj value: 0.5918920327236725 - filter: model.layers.19.mlp.down_proj value: 0.6033557396696368 - filter: model.layers.20.mlp.down_proj value: 0.5831900369965088 - filter: model.layers.21.mlp.down_proj value: 0.6060392892501694 - filter: model.layers.22.mlp.down_proj value: 0.5592465218070971 - filter: model.layers.23.mlp.down_proj value: 0.5628159032879996 - filter: model.layers.24.mlp.down_proj value: 0.5611484550049503 - filter: model.layers.25.mlp.down_proj value: 0.6383460997342505 - filter: model.layers.26.mlp.down_proj value: 0.5379865561982179 - filter: model.layers.27.mlp.down_proj value: 0.547183575634412 - filter: model.layers.28.mlp.down_proj value: 0.5039862435516649 - filter: model.layers.29.mlp.down_proj value: 0.8178312750768589 - filter: model.layers.30.mlp.down_proj value: 0.7657756239904121 - filter: model.layers.31.mlp.down_proj value: 0.8403939346568705 - value: 1.0 weight: - value: 1.0 tokenizer_source: union ```