choprahetarth/llama-it-merged

Untitled Model (1)

This is a merge of pre-trained language models created using mergekit.

Merge Details

Merge Method

This model was merged using the TIES merge method using meta-llama/Meta-Llama-3-8B as a base.

Models Merged

The following models were included in the merge:

• meta-llama/Meta-Llama-3-8B-Instruct

Configuration

The following YAML configuration was used to produce this model:

base_model: meta-llama/Meta-Llama-3-8B
dtype: bfloat16
merge_method: ties
parameters:
  int8_mask: 1.0
  normalize: 1.0
slices:
- sources:
  - layer_range: [0, 32]
    model: meta-llama/Meta-Llama-3-8B
  - layer_range: [0, 32]
    model: meta-llama/Meta-Llama-3-8B-Instruct
    parameters:
      density:
      - filter: model.layers.0.self_attn.q_proj
        value: 1.0
      - filter: model.layers.1.self_attn.q_proj
        value: 0.5915012245323329
      - filter: model.layers.2.self_attn.q_proj
        value: 0.533635558334636
      - filter: model.layers.3.self_attn.q_proj
        value: 0.5581262049919233
      - filter: model.layers.4.self_attn.q_proj
        value: 0.5773018602469908
      - filter: model.layers.5.self_attn.q_proj
        value: 0.5594288989630556
      - filter: model.layers.6.self_attn.q_proj
        value: 0.5264707414934083
      - filter: model.layers.7.self_attn.q_proj
        value: 0.5074253556354541
      - filter: model.layers.8.self_attn.q_proj
        value: 0.5123234849669116
      - filter: model.layers.9.self_attn.q_proj
        value: 0.549606586420718
      - filter: model.layers.10.self_attn.q_proj
        value: 0.5104736595279037
      - filter: model.layers.11.self_attn.q_proj
        value: 0.521937366473868
      - filter: model.layers.12.self_attn.q_proj
        value: 0.5211036423323433
      - filter: model.layers.13.self_attn.q_proj
        value: 0.5566932416236777
      - filter: model.layers.14.self_attn.q_proj
        value: 0.5287113751237559
      - filter: model.layers.15.self_attn.q_proj
        value: 0.5587254442186441
      - filter: model.layers.16.self_attn.q_proj
        value: 0.5647699442446981
      - filter: model.layers.17.self_attn.q_proj
        value: 0.5868636339951019
      - filter: model.layers.18.self_attn.q_proj
        value: 0.590172476681778
      - filter: model.layers.19.self_attn.q_proj
        value: 0.5772236986087228
      - filter: model.layers.20.self_attn.q_proj
        value: 0.5837371684643844
      - filter: model.layers.21.self_attn.q_proj
        value: 0.5988484185295191
      - filter: model.layers.22.self_attn.q_proj
        value: 0.5956177374811109
      - filter: model.layers.23.self_attn.q_proj
        value: 0.6109113647022042
      - filter: model.layers.24.self_attn.q_proj
        value: 0.607680683653796
      - filter: model.layers.25.self_attn.q_proj
        value: 0.612396435829295
      - filter: model.layers.26.self_attn.q_proj
        value: 0.655307175238393
      - filter: model.layers.27.self_attn.q_proj
        value: 0.6167734875722996
      - filter: model.layers.28.self_attn.q_proj
        value: 0.6221405867333646
      - filter: model.layers.29.self_attn.q_proj
        value: 0.5649783752800792
      - filter: model.layers.30.self_attn.q_proj
        value: 0.5978323172320359
      - filter: model.layers.31.self_attn.q_proj
        value: 0.5
      - filter: model.layers.0.self_attn.k_proj
        value: 0.8480537752071284
      - filter: model.layers.1.self_attn.k_proj
        value: 0.5569537804179042
      - filter: model.layers.2.self_attn.k_proj
        value: 0.5319160022927414
      - filter: model.layers.3.self_attn.k_proj
        value: 0.5251680475222761
      - filter: model.layers.4.self_attn.k_proj
        value: 0.5493460476264915
      - filter: model.layers.5.self_attn.k_proj
        value: 0.5465843364076911
      - filter: model.layers.6.self_attn.k_proj
        value: 0.5242301078630608
      - filter: model.layers.7.self_attn.k_proj
        value: 0.5145901724766818
      - filter: model.layers.8.self_attn.k_proj
        value: 0.5039862435516649
      - filter: model.layers.9.self_attn.k_proj
        value: 0.544526079933302
      - filter: model.layers.10.self_attn.k_proj
        value: 0.5103694440102131
      - filter: model.layers.11.self_attn.k_proj
        value: 0.5106560366838623
      - filter: model.layers.12.self_attn.k_proj
        value: 0.5145641185972591
      - filter: model.layers.13.self_attn.k_proj
        value: 0.5530717523839299
      - filter: model.layers.14.self_attn.k_proj
        value: 0.5418685842321922
      - filter: model.layers.15.self_attn.k_proj
        value: 0.5876713042572039
      - filter: model.layers.16.self_attn.k_proj
        value: 0.6299046428013131
      - filter: model.layers.17.self_attn.k_proj
        value: 0.6432963368245532
      - filter: model.layers.18.self_attn.k_proj
        value: 0.6294877807305508
      - filter: model.layers.19.self_attn.k_proj
        value: 0.604293679328852
      - filter: model.layers.20.self_attn.k_proj
        value: 0.6142723151477255
      - filter: model.layers.21.self_attn.k_proj
        value: 0.6172424574019072
      - filter: model.layers.22.self_attn.k_proj
        value: 0.627559793653275
      - filter: model.layers.23.self_attn.k_proj
        value: 0.6269345005471315
      - filter: model.layers.24.self_attn.k_proj
        value: 0.6253973216611954
      - filter: model.layers.25.self_attn.k_proj
        value: 0.6097128862487624
      - filter: model.layers.26.self_attn.k_proj
        value: 0.654499504976291
      - filter: model.layers.27.self_attn.k_proj
        value: 0.6062737741649732
      - filter: model.layers.28.self_attn.k_proj
        value: 0.6051534573497994
      - filter: model.layers.29.self_attn.k_proj
        value: 0.5576832890417384
      - filter: model.layers.30.self_attn.k_proj
        value: 0.5832942525141994
      - filter: model.layers.31.self_attn.k_proj
        value: 0.5001302693971132
      - filter: model.layers.0.self_attn.v_proj
        value: 0.6780001042155177
      - filter: model.layers.1.self_attn.v_proj
        value: 0.5999687353446929
      - filter: model.layers.2.self_attn.v_proj
        value: 0.5700067740086499
      - filter: model.layers.3.self_attn.v_proj
        value: 0.633447970402793
      - filter: model.layers.4.self_attn.v_proj
        value: 0.6278203324475015
      - filter: model.layers.5.self_attn.v_proj
        value: 0.5794382783596478
      - filter: model.layers.6.self_attn.v_proj
        value: 0.5442655411390755
      - filter: model.layers.7.self_attn.v_proj
        value: 0.5154499504976291
      - filter: model.layers.8.self_attn.v_proj
        value: 0.5289719139179824
      - filter: model.layers.9.self_attn.v_proj
        value: 0.522302120785785
      - filter: model.layers.10.self_attn.v_proj
        value: 0.5297795841800844
      - filter: model.layers.11.self_attn.v_proj
        value: 0.5429367932885206
      - filter: model.layers.12.self_attn.v_proj
        value: 0.5740451253191601
      - filter: model.layers.13.self_attn.v_proj
        value: 0.5515866812568392
      - filter: model.layers.14.self_attn.v_proj
        value: 0.5319420561721641
      - filter: model.layers.15.self_attn.v_proj
        value: 0.6064040435620864
      - filter: model.layers.16.self_attn.v_proj
        value: 0.6014538064717837
      - filter: model.layers.17.self_attn.v_proj
        value: 0.6136730759210046
      - filter: model.layers.18.self_attn.v_proj
        value: 0.6215152936272211
      - filter: model.layers.19.self_attn.v_proj
        value: 0.6304517742691886
      - filter: model.layers.20.self_attn.v_proj
        value: 0.6145849617007972
      - filter: model.layers.21.self_attn.v_proj
        value: 0.6035120629461727
      - filter: model.layers.22.self_attn.v_proj
        value: 0.624537543640248
      - filter: model.layers.23.self_attn.v_proj
        value: 0.6480641967588974
      - filter: model.layers.24.self_attn.v_proj
        value: 0.6352456880829556
      - filter: model.layers.25.self_attn.v_proj
        value: 0.5832681986347767
      - filter: model.layers.26.self_attn.v_proj
        value: 0.6047887030378823
      - filter: model.layers.27.self_attn.v_proj
        value: 0.5266010108905216
      - filter: model.layers.28.self_attn.v_proj
        value: 0.5026835495805325
      - filter: model.layers.29.self_attn.v_proj
        value: 0.5128966703142098
      - filter: model.layers.30.self_attn.v_proj
        value: 0.559038090771716
      - filter: model.layers.31.self_attn.v_proj
        value: 0.580011463706946
      - filter: model.layers.0.self_attn.o_proj
        value: 0.5566150799854098
      - filter: model.layers.1.self_attn.o_proj
        value: 0.516335782397999
      - filter: model.layers.2.self_attn.o_proj
        value: 0.5357980303267157
      - filter: model.layers.3.self_attn.o_proj
        value: 0.5400187587931843
      - filter: model.layers.4.self_attn.o_proj
        value: 0.6263613151998333
      - filter: model.layers.5.self_attn.o_proj
        value: 0.5280339742587671
      - filter: model.layers.6.self_attn.o_proj
        value: 0.5044812672606951
      - filter: model.layers.7.self_attn.o_proj
        value: 0.5143296336824553
      - filter: model.layers.8.self_attn.o_proj
        value: 0.5011984784534417
      - filter: model.layers.9.self_attn.o_proj
        value: 0.5095878276275337
      - filter: model.layers.10.self_attn.o_proj
        value: 0.558595174821531
      - filter: model.layers.11.self_attn.o_proj
        value: 0.5168047522276067
      - filter: model.layers.12.self_attn.o_proj
        value: 0.586264394768381
      - filter: model.layers.13.self_attn.o_proj
        value: 0.5044812672606951
      - filter: model.layers.14.self_attn.o_proj
        value: 0.5224844979417436
      - filter: model.layers.15.self_attn.o_proj
        value: 0.512584023761138
      - filter: model.layers.16.self_attn.o_proj
        value: 0.5946016361836277
      - filter: model.layers.17.self_attn.o_proj
        value: 0.5662289614923662
      - filter: model.layers.18.self_attn.o_proj
        value: 0.5052368297639519
      - filter: model.layers.19.self_attn.o_proj
        value: 0.5294669376270127
      - filter: model.layers.20.self_attn.o_proj
        value: 0.6374602678338804
      - filter: model.layers.21.self_attn.o_proj
        value: 0.583763222343807
      - filter: model.layers.22.self_attn.o_proj
        value: 0.694752748684279
      - filter: model.layers.23.self_attn.o_proj
        value: 0.5834505757907352
      - filter: model.layers.24.self_attn.o_proj
        value: 0.57493095721953
      - filter: model.layers.25.self_attn.o_proj
        value: 0.9771507477463395
      - filter: model.layers.26.self_attn.o_proj
        value: 0.5683132718461779
      - filter: model.layers.27.self_attn.o_proj
        value: 0.8806992861237039
      - filter: model.layers.28.self_attn.o_proj
        value: 0.585013808556094
      - filter: model.layers.29.self_attn.o_proj
        value: 0.7172111927466
      - filter: model.layers.30.self_attn.o_proj
        value: 0.539158980772237
      - filter: model.layers.31.self_attn.o_proj
        value: 0.5273565733937783
      - filter: model.layers.0.mlp.gate_proj
        value: 0.7398259600854568
      - filter: model.layers.1.mlp.gate_proj
        value: 0.6623156688030848
      - filter: model.layers.2.mlp.gate_proj
        value: 0.6218800479391382
      - filter: model.layers.3.mlp.gate_proj
        value: 0.6496795372831015
      - filter: model.layers.4.mlp.gate_proj
        value: 0.6073419832213016
      - filter: model.layers.5.mlp.gate_proj
        value: 0.5901203689229326
      - filter: model.layers.6.mlp.gate_proj
        value: 0.5831639831170862
      - filter: model.layers.7.mlp.gate_proj
        value: 0.5327757803136887
      - filter: model.layers.8.mlp.gate_proj
        value: 0.5048460215726122
      - filter: model.layers.9.mlp.gate_proj
        value: 0.517299775936637
      - filter: model.layers.10.mlp.gate_proj
        value: 0.5256109634724611
      - filter: model.layers.11.mlp.gate_proj
        value: 0.5363191079151686
      - filter: model.layers.12.mlp.gate_proj
        value: 0.5258454483872649
      - filter: model.layers.13.mlp.gate_proj
        value: 0.5099004741806055
      - filter: model.layers.14.mlp.gate_proj
        value: 0.544161325621385
      - filter: model.layers.15.mlp.gate_proj
        value: 0.5567974571413683
      - filter: model.layers.16.mlp.gate_proj
        value: 0.5865249335626075
      - filter: model.layers.17.mlp.gate_proj
        value: 0.6023917461309989
      - filter: model.layers.18.mlp.gate_proj
        value: 0.6177114272315147
      - filter: model.layers.19.mlp.gate_proj
        value: 0.614011776353499
      - filter: model.layers.20.mlp.gate_proj
        value: 0.6049189724349956
      - filter: model.layers.21.mlp.gate_proj
        value: 0.5991871189620135
      - filter: model.layers.22.mlp.gate_proj
        value: 0.6197957375853265
      - filter: model.layers.23.mlp.gate_proj
        value: 0.6167734875722996
      - filter: model.layers.24.mlp.gate_proj
        value: 0.5977281017143453
      - filter: model.layers.25.mlp.gate_proj
        value: 0.6520764941899849
      - filter: model.layers.26.mlp.gate_proj
        value: 0.5683653796050232
      - filter: model.layers.27.mlp.gate_proj
        value: 0.5564587567088739
      - filter: model.layers.28.mlp.gate_proj
        value: 0.5045333750195404
      - filter: model.layers.29.mlp.gate_proj
        value: 0.5245427544161325
      - filter: model.layers.30.mlp.gate_proj
        value: 0.5658902610598718
      - filter: model.layers.31.mlp.gate_proj
        value: 0.5598978687926632
      - filter: model.layers.0.mlp.up_proj
        value: 0.7446198738992236
      - filter: model.layers.1.mlp.up_proj
        value: 0.6871971236517117
      - filter: model.layers.2.mlp.up_proj
        value: 0.6360012505862123
      - filter: model.layers.3.mlp.up_proj
        value: 0.6525194101401699
      - filter: model.layers.4.mlp.up_proj
        value: 0.5973112396435829
      - filter: model.layers.5.mlp.up_proj
        value: 0.5818873430253765
      - filter: model.layers.6.mlp.up_proj
        value: 0.5714397373768955
      - filter: model.layers.7.mlp.up_proj
        value: 0.5230576832890418
      - filter: model.layers.8.mlp.up_proj
        value: 0.5084935646917826
      - filter: model.layers.9.mlp.up_proj
        value: 0.5306393622010317
      - filter: model.layers.10.mlp.up_proj
        value: 0.5488249700380387
      - filter: model.layers.11.mlp.up_proj
        value: 0.5646657287270075
      - filter: model.layers.12.mlp.up_proj
        value: 0.557266426970976
      - filter: model.layers.13.mlp.up_proj
        value: 0.5164399979156896
      - filter: model.layers.14.mlp.up_proj
        value: 0.5224584440623209
      - filter: model.layers.15.mlp.up_proj
        value: 0.5427283622531395
      - filter: model.layers.16.mlp.up_proj
        value: 0.5844927309676411
      - filter: model.layers.17.mlp.up_proj
        value: 0.5983012870616435
      - filter: model.layers.18.mlp.up_proj
        value: 0.6132301599708196
      - filter: model.layers.19.mlp.up_proj
        value: 0.6065603668386222
      - filter: model.layers.20.mlp.up_proj
        value: 0.598040748267417
      - filter: model.layers.21.mlp.up_proj
        value: 0.5880360585691209
      - filter: model.layers.22.mlp.up_proj
        value: 0.6064561513209317
      - filter: model.layers.23.mlp.up_proj
        value: 0.6022093689750404
      - filter: model.layers.24.mlp.up_proj
        value: 0.580376218018863
      - filter: model.layers.25.mlp.up_proj
        value: 0.6365223281746653
      - filter: model.layers.26.mlp.up_proj
        value: 0.5579698817153874
      - filter: model.layers.27.mlp.up_proj
        value: 0.5470272523578761
      - filter: model.layers.28.mlp.up_proj
        value: 0.5102131207336772
      - filter: model.layers.29.mlp.up_proj
        value: 0.5205044031056224
      - filter: model.layers.30.mlp.up_proj
        value: 0.555651086446772
      - filter: model.layers.31.mlp.up_proj
        value: 0.6076025220155281
      - filter: model.layers.0.mlp.down_proj
        value: 0.5857954249387733
      - filter: model.layers.1.mlp.down_proj
        value: 0.5853264551091657
      - filter: model.layers.2.mlp.down_proj
        value: 0.5342087436819343
      - filter: model.layers.3.mlp.down_proj
        value: 0.5606534312959199
      - filter: model.layers.4.mlp.down_proj
        value: 0.566541608045438
      - filter: model.layers.5.mlp.down_proj
        value: 0.5318899484133187
      - filter: model.layers.6.mlp.down_proj
        value: 0.538351310510135
      - filter: model.layers.7.mlp.down_proj
        value: 0.5209994268146527
      - filter: model.layers.8.mlp.down_proj
        value: 0.5305351466833411
      - filter: model.layers.9.mlp.down_proj
        value: 0.503673596998593
      - filter: model.layers.10.mlp.down_proj
        value: 0.5296232609035485
      - filter: model.layers.11.mlp.down_proj
        value: 0.5282163514147257
      - filter: model.layers.12.mlp.down_proj
        value: 0.549606586420718
      - filter: model.layers.13.mlp.down_proj
        value: 0.5241519462247929
      - filter: model.layers.14.mlp.down_proj
        value: 0.5465582825282684
      - filter: model.layers.15.mlp.down_proj
        value: 0.5839977072586108
      - filter: model.layers.16.mlp.down_proj
        value: 0.5891824292637173
      - filter: model.layers.17.mlp.down_proj
        value: 0.570215205044031
      - filter: model.layers.18.mlp.down_proj
        value: 0.5918920327236725
      - filter: model.layers.19.mlp.down_proj
        value: 0.6033557396696368
      - filter: model.layers.20.mlp.down_proj
        value: 0.5831900369965088
      - filter: model.layers.21.mlp.down_proj
        value: 0.6060392892501694
      - filter: model.layers.22.mlp.down_proj
        value: 0.5592465218070971
      - filter: model.layers.23.mlp.down_proj
        value: 0.5628159032879996
      - filter: model.layers.24.mlp.down_proj
        value: 0.5611484550049503
      - filter: model.layers.25.mlp.down_proj
        value: 0.6383460997342505
      - filter: model.layers.26.mlp.down_proj
        value: 0.5379865561982179
      - filter: model.layers.27.mlp.down_proj
        value: 0.547183575634412
      - filter: model.layers.28.mlp.down_proj
        value: 0.5039862435516649
      - filter: model.layers.29.mlp.down_proj
        value: 0.8178312750768589
      - filter: model.layers.30.mlp.down_proj
        value: 0.7657756239904121
      - filter: model.layers.31.mlp.down_proj
        value: 0.8403939346568705
      - value: 1.0
      weight:
      - value: 1.0
tokenizer_source: union