metadata
base_model:
- meta-llama/Meta-Llama-3-8B
- meta-llama/Meta-Llama-3-8B-Instruct
library_name: transformers
tags:
- mergekit
- merge
Untitled Model (1)
This is a merge of pre-trained language models created using mergekit.
Merge Details
Merge Method
This model was merged using the TIES merge method using meta-llama/Meta-Llama-3-8B as a base.
Models Merged
The following models were included in the merge:
Configuration
The following YAML configuration was used to produce this model:
base_model: meta-llama/Meta-Llama-3-8B
dtype: bfloat16
merge_method: ties
parameters:
int8_mask: 1.0
normalize: 1.0
slices:
- sources:
- layer_range: [0, 32]
model: meta-llama/Meta-Llama-3-8B
- layer_range: [0, 32]
model: meta-llama/Meta-Llama-3-8B-Instruct
parameters:
density:
- filter: model.layers.0.self_attn.q_proj
value: 1.0
- filter: model.layers.1.self_attn.q_proj
value: 0.5915012245323329
- filter: model.layers.2.self_attn.q_proj
value: 0.533635558334636
- filter: model.layers.3.self_attn.q_proj
value: 0.5581262049919233
- filter: model.layers.4.self_attn.q_proj
value: 0.5773018602469908
- filter: model.layers.5.self_attn.q_proj
value: 0.5594288989630556
- filter: model.layers.6.self_attn.q_proj
value: 0.5264707414934083
- filter: model.layers.7.self_attn.q_proj
value: 0.5074253556354541
- filter: model.layers.8.self_attn.q_proj
value: 0.5123234849669116
- filter: model.layers.9.self_attn.q_proj
value: 0.549606586420718
- filter: model.layers.10.self_attn.q_proj
value: 0.5104736595279037
- filter: model.layers.11.self_attn.q_proj
value: 0.521937366473868
- filter: model.layers.12.self_attn.q_proj
value: 0.5211036423323433
- filter: model.layers.13.self_attn.q_proj
value: 0.5566932416236777
- filter: model.layers.14.self_attn.q_proj
value: 0.5287113751237559
- filter: model.layers.15.self_attn.q_proj
value: 0.5587254442186441
- filter: model.layers.16.self_attn.q_proj
value: 0.5647699442446981
- filter: model.layers.17.self_attn.q_proj
value: 0.5868636339951019
- filter: model.layers.18.self_attn.q_proj
value: 0.590172476681778
- filter: model.layers.19.self_attn.q_proj
value: 0.5772236986087228
- filter: model.layers.20.self_attn.q_proj
value: 0.5837371684643844
- filter: model.layers.21.self_attn.q_proj
value: 0.5988484185295191
- filter: model.layers.22.self_attn.q_proj
value: 0.5956177374811109
- filter: model.layers.23.self_attn.q_proj
value: 0.6109113647022042
- filter: model.layers.24.self_attn.q_proj
value: 0.607680683653796
- filter: model.layers.25.self_attn.q_proj
value: 0.612396435829295
- filter: model.layers.26.self_attn.q_proj
value: 0.655307175238393
- filter: model.layers.27.self_attn.q_proj
value: 0.6167734875722996
- filter: model.layers.28.self_attn.q_proj
value: 0.6221405867333646
- filter: model.layers.29.self_attn.q_proj
value: 0.5649783752800792
- filter: model.layers.30.self_attn.q_proj
value: 0.5978323172320359
- filter: model.layers.31.self_attn.q_proj
value: 0.5
- filter: model.layers.0.self_attn.k_proj
value: 0.8480537752071284
- filter: model.layers.1.self_attn.k_proj
value: 0.5569537804179042
- filter: model.layers.2.self_attn.k_proj
value: 0.5319160022927414
- filter: model.layers.3.self_attn.k_proj
value: 0.5251680475222761
- filter: model.layers.4.self_attn.k_proj
value: 0.5493460476264915
- filter: model.layers.5.self_attn.k_proj
value: 0.5465843364076911
- filter: model.layers.6.self_attn.k_proj
value: 0.5242301078630608
- filter: model.layers.7.self_attn.k_proj
value: 0.5145901724766818
- filter: model.layers.8.self_attn.k_proj
value: 0.5039862435516649
- filter: model.layers.9.self_attn.k_proj
value: 0.544526079933302
- filter: model.layers.10.self_attn.k_proj
value: 0.5103694440102131
- filter: model.layers.11.self_attn.k_proj
value: 0.5106560366838623
- filter: model.layers.12.self_attn.k_proj
value: 0.5145641185972591
- filter: model.layers.13.self_attn.k_proj
value: 0.5530717523839299
- filter: model.layers.14.self_attn.k_proj
value: 0.5418685842321922
- filter: model.layers.15.self_attn.k_proj
value: 0.5876713042572039
- filter: model.layers.16.self_attn.k_proj
value: 0.6299046428013131
- filter: model.layers.17.self_attn.k_proj
value: 0.6432963368245532
- filter: model.layers.18.self_attn.k_proj
value: 0.6294877807305508
- filter: model.layers.19.self_attn.k_proj
value: 0.604293679328852
- filter: model.layers.20.self_attn.k_proj
value: 0.6142723151477255
- filter: model.layers.21.self_attn.k_proj
value: 0.6172424574019072
- filter: model.layers.22.self_attn.k_proj
value: 0.627559793653275
- filter: model.layers.23.self_attn.k_proj
value: 0.6269345005471315
- filter: model.layers.24.self_attn.k_proj
value: 0.6253973216611954
- filter: model.layers.25.self_attn.k_proj
value: 0.6097128862487624
- filter: model.layers.26.self_attn.k_proj
value: 0.654499504976291
- filter: model.layers.27.self_attn.k_proj
value: 0.6062737741649732
- filter: model.layers.28.self_attn.k_proj
value: 0.6051534573497994
- filter: model.layers.29.self_attn.k_proj
value: 0.5576832890417384
- filter: model.layers.30.self_attn.k_proj
value: 0.5832942525141994
- filter: model.layers.31.self_attn.k_proj
value: 0.5001302693971132
- filter: model.layers.0.self_attn.v_proj
value: 0.6780001042155177
- filter: model.layers.1.self_attn.v_proj
value: 0.5999687353446929
- filter: model.layers.2.self_attn.v_proj
value: 0.5700067740086499
- filter: model.layers.3.self_attn.v_proj
value: 0.633447970402793
- filter: model.layers.4.self_attn.v_proj
value: 0.6278203324475015
- filter: model.layers.5.self_attn.v_proj
value: 0.5794382783596478
- filter: model.layers.6.self_attn.v_proj
value: 0.5442655411390755
- filter: model.layers.7.self_attn.v_proj
value: 0.5154499504976291
- filter: model.layers.8.self_attn.v_proj
value: 0.5289719139179824
- filter: model.layers.9.self_attn.v_proj
value: 0.522302120785785
- filter: model.layers.10.self_attn.v_proj
value: 0.5297795841800844
- filter: model.layers.11.self_attn.v_proj
value: 0.5429367932885206
- filter: model.layers.12.self_attn.v_proj
value: 0.5740451253191601
- filter: model.layers.13.self_attn.v_proj
value: 0.5515866812568392
- filter: model.layers.14.self_attn.v_proj
value: 0.5319420561721641
- filter: model.layers.15.self_attn.v_proj
value: 0.6064040435620864
- filter: model.layers.16.self_attn.v_proj
value: 0.6014538064717837
- filter: model.layers.17.self_attn.v_proj
value: 0.6136730759210046
- filter: model.layers.18.self_attn.v_proj
value: 0.6215152936272211
- filter: model.layers.19.self_attn.v_proj
value: 0.6304517742691886
- filter: model.layers.20.self_attn.v_proj
value: 0.6145849617007972
- filter: model.layers.21.self_attn.v_proj
value: 0.6035120629461727
- filter: model.layers.22.self_attn.v_proj
value: 0.624537543640248
- filter: model.layers.23.self_attn.v_proj
value: 0.6480641967588974
- filter: model.layers.24.self_attn.v_proj
value: 0.6352456880829556
- filter: model.layers.25.self_attn.v_proj
value: 0.5832681986347767
- filter: model.layers.26.self_attn.v_proj
value: 0.6047887030378823
- filter: model.layers.27.self_attn.v_proj
value: 0.5266010108905216
- filter: model.layers.28.self_attn.v_proj
value: 0.5026835495805325
- filter: model.layers.29.self_attn.v_proj
value: 0.5128966703142098
- filter: model.layers.30.self_attn.v_proj
value: 0.559038090771716
- filter: model.layers.31.self_attn.v_proj
value: 0.580011463706946
- filter: model.layers.0.self_attn.o_proj
value: 0.5566150799854098
- filter: model.layers.1.self_attn.o_proj
value: 0.516335782397999
- filter: model.layers.2.self_attn.o_proj
value: 0.5357980303267157
- filter: model.layers.3.self_attn.o_proj
value: 0.5400187587931843
- filter: model.layers.4.self_attn.o_proj
value: 0.6263613151998333
- filter: model.layers.5.self_attn.o_proj
value: 0.5280339742587671
- filter: model.layers.6.self_attn.o_proj
value: 0.5044812672606951
- filter: model.layers.7.self_attn.o_proj
value: 0.5143296336824553
- filter: model.layers.8.self_attn.o_proj
value: 0.5011984784534417
- filter: model.layers.9.self_attn.o_proj
value: 0.5095878276275337
- filter: model.layers.10.self_attn.o_proj
value: 0.558595174821531
- filter: model.layers.11.self_attn.o_proj
value: 0.5168047522276067
- filter: model.layers.12.self_attn.o_proj
value: 0.586264394768381
- filter: model.layers.13.self_attn.o_proj
value: 0.5044812672606951
- filter: model.layers.14.self_attn.o_proj
value: 0.5224844979417436
- filter: model.layers.15.self_attn.o_proj
value: 0.512584023761138
- filter: model.layers.16.self_attn.o_proj
value: 0.5946016361836277
- filter: model.layers.17.self_attn.o_proj
value: 0.5662289614923662
- filter: model.layers.18.self_attn.o_proj
value: 0.5052368297639519
- filter: model.layers.19.self_attn.o_proj
value: 0.5294669376270127
- filter: model.layers.20.self_attn.o_proj
value: 0.6374602678338804
- filter: model.layers.21.self_attn.o_proj
value: 0.583763222343807
- filter: model.layers.22.self_attn.o_proj
value: 0.694752748684279
- filter: model.layers.23.self_attn.o_proj
value: 0.5834505757907352
- filter: model.layers.24.self_attn.o_proj
value: 0.57493095721953
- filter: model.layers.25.self_attn.o_proj
value: 0.9771507477463395
- filter: model.layers.26.self_attn.o_proj
value: 0.5683132718461779
- filter: model.layers.27.self_attn.o_proj
value: 0.8806992861237039
- filter: model.layers.28.self_attn.o_proj
value: 0.585013808556094
- filter: model.layers.29.self_attn.o_proj
value: 0.7172111927466
- filter: model.layers.30.self_attn.o_proj
value: 0.539158980772237
- filter: model.layers.31.self_attn.o_proj
value: 0.5273565733937783
- filter: model.layers.0.mlp.gate_proj
value: 0.7398259600854568
- filter: model.layers.1.mlp.gate_proj
value: 0.6623156688030848
- filter: model.layers.2.mlp.gate_proj
value: 0.6218800479391382
- filter: model.layers.3.mlp.gate_proj
value: 0.6496795372831015
- filter: model.layers.4.mlp.gate_proj
value: 0.6073419832213016
- filter: model.layers.5.mlp.gate_proj
value: 0.5901203689229326
- filter: model.layers.6.mlp.gate_proj
value: 0.5831639831170862
- filter: model.layers.7.mlp.gate_proj
value: 0.5327757803136887
- filter: model.layers.8.mlp.gate_proj
value: 0.5048460215726122
- filter: model.layers.9.mlp.gate_proj
value: 0.517299775936637
- filter: model.layers.10.mlp.gate_proj
value: 0.5256109634724611
- filter: model.layers.11.mlp.gate_proj
value: 0.5363191079151686
- filter: model.layers.12.mlp.gate_proj
value: 0.5258454483872649
- filter: model.layers.13.mlp.gate_proj
value: 0.5099004741806055
- filter: model.layers.14.mlp.gate_proj
value: 0.544161325621385
- filter: model.layers.15.mlp.gate_proj
value: 0.5567974571413683
- filter: model.layers.16.mlp.gate_proj
value: 0.5865249335626075
- filter: model.layers.17.mlp.gate_proj
value: 0.6023917461309989
- filter: model.layers.18.mlp.gate_proj
value: 0.6177114272315147
- filter: model.layers.19.mlp.gate_proj
value: 0.614011776353499
- filter: model.layers.20.mlp.gate_proj
value: 0.6049189724349956
- filter: model.layers.21.mlp.gate_proj
value: 0.5991871189620135
- filter: model.layers.22.mlp.gate_proj
value: 0.6197957375853265
- filter: model.layers.23.mlp.gate_proj
value: 0.6167734875722996
- filter: model.layers.24.mlp.gate_proj
value: 0.5977281017143453
- filter: model.layers.25.mlp.gate_proj
value: 0.6520764941899849
- filter: model.layers.26.mlp.gate_proj
value: 0.5683653796050232
- filter: model.layers.27.mlp.gate_proj
value: 0.5564587567088739
- filter: model.layers.28.mlp.gate_proj
value: 0.5045333750195404
- filter: model.layers.29.mlp.gate_proj
value: 0.5245427544161325
- filter: model.layers.30.mlp.gate_proj
value: 0.5658902610598718
- filter: model.layers.31.mlp.gate_proj
value: 0.5598978687926632
- filter: model.layers.0.mlp.up_proj
value: 0.7446198738992236
- filter: model.layers.1.mlp.up_proj
value: 0.6871971236517117
- filter: model.layers.2.mlp.up_proj
value: 0.6360012505862123
- filter: model.layers.3.mlp.up_proj
value: 0.6525194101401699
- filter: model.layers.4.mlp.up_proj
value: 0.5973112396435829
- filter: model.layers.5.mlp.up_proj
value: 0.5818873430253765
- filter: model.layers.6.mlp.up_proj
value: 0.5714397373768955
- filter: model.layers.7.mlp.up_proj
value: 0.5230576832890418
- filter: model.layers.8.mlp.up_proj
value: 0.5084935646917826
- filter: model.layers.9.mlp.up_proj
value: 0.5306393622010317
- filter: model.layers.10.mlp.up_proj
value: 0.5488249700380387
- filter: model.layers.11.mlp.up_proj
value: 0.5646657287270075
- filter: model.layers.12.mlp.up_proj
value: 0.557266426970976
- filter: model.layers.13.mlp.up_proj
value: 0.5164399979156896
- filter: model.layers.14.mlp.up_proj
value: 0.5224584440623209
- filter: model.layers.15.mlp.up_proj
value: 0.5427283622531395
- filter: model.layers.16.mlp.up_proj
value: 0.5844927309676411
- filter: model.layers.17.mlp.up_proj
value: 0.5983012870616435
- filter: model.layers.18.mlp.up_proj
value: 0.6132301599708196
- filter: model.layers.19.mlp.up_proj
value: 0.6065603668386222
- filter: model.layers.20.mlp.up_proj
value: 0.598040748267417
- filter: model.layers.21.mlp.up_proj
value: 0.5880360585691209
- filter: model.layers.22.mlp.up_proj
value: 0.6064561513209317
- filter: model.layers.23.mlp.up_proj
value: 0.6022093689750404
- filter: model.layers.24.mlp.up_proj
value: 0.580376218018863
- filter: model.layers.25.mlp.up_proj
value: 0.6365223281746653
- filter: model.layers.26.mlp.up_proj
value: 0.5579698817153874
- filter: model.layers.27.mlp.up_proj
value: 0.5470272523578761
- filter: model.layers.28.mlp.up_proj
value: 0.5102131207336772
- filter: model.layers.29.mlp.up_proj
value: 0.5205044031056224
- filter: model.layers.30.mlp.up_proj
value: 0.555651086446772
- filter: model.layers.31.mlp.up_proj
value: 0.6076025220155281
- filter: model.layers.0.mlp.down_proj
value: 0.5857954249387733
- filter: model.layers.1.mlp.down_proj
value: 0.5853264551091657
- filter: model.layers.2.mlp.down_proj
value: 0.5342087436819343
- filter: model.layers.3.mlp.down_proj
value: 0.5606534312959199
- filter: model.layers.4.mlp.down_proj
value: 0.566541608045438
- filter: model.layers.5.mlp.down_proj
value: 0.5318899484133187
- filter: model.layers.6.mlp.down_proj
value: 0.538351310510135
- filter: model.layers.7.mlp.down_proj
value: 0.5209994268146527
- filter: model.layers.8.mlp.down_proj
value: 0.5305351466833411
- filter: model.layers.9.mlp.down_proj
value: 0.503673596998593
- filter: model.layers.10.mlp.down_proj
value: 0.5296232609035485
- filter: model.layers.11.mlp.down_proj
value: 0.5282163514147257
- filter: model.layers.12.mlp.down_proj
value: 0.549606586420718
- filter: model.layers.13.mlp.down_proj
value: 0.5241519462247929
- filter: model.layers.14.mlp.down_proj
value: 0.5465582825282684
- filter: model.layers.15.mlp.down_proj
value: 0.5839977072586108
- filter: model.layers.16.mlp.down_proj
value: 0.5891824292637173
- filter: model.layers.17.mlp.down_proj
value: 0.570215205044031
- filter: model.layers.18.mlp.down_proj
value: 0.5918920327236725
- filter: model.layers.19.mlp.down_proj
value: 0.6033557396696368
- filter: model.layers.20.mlp.down_proj
value: 0.5831900369965088
- filter: model.layers.21.mlp.down_proj
value: 0.6060392892501694
- filter: model.layers.22.mlp.down_proj
value: 0.5592465218070971
- filter: model.layers.23.mlp.down_proj
value: 0.5628159032879996
- filter: model.layers.24.mlp.down_proj
value: 0.5611484550049503
- filter: model.layers.25.mlp.down_proj
value: 0.6383460997342505
- filter: model.layers.26.mlp.down_proj
value: 0.5379865561982179
- filter: model.layers.27.mlp.down_proj
value: 0.547183575634412
- filter: model.layers.28.mlp.down_proj
value: 0.5039862435516649
- filter: model.layers.29.mlp.down_proj
value: 0.8178312750768589
- filter: model.layers.30.mlp.down_proj
value: 0.7657756239904121
- filter: model.layers.31.mlp.down_proj
value: 0.8403939346568705
- value: 1.0
weight:
- value: 1.0
tokenizer_source: union