metadata
base_model:
- meta-llama/Meta-Llama-3-8B
- meta-llama/Meta-Llama-3-8B-Instruct
library_name: transformers
tags:
- mergekit
- merge
Llama-3-EvoLLM-v2-8B
This is a merge of pre-trained language models created using mergekit.
Merge Details
Merge Method
This model was merged using the DARE TIES merge method using meta-llama/Meta-Llama-3-8B as a base.
Models Merged
The following models were included in the merge:
Configuration
The following YAML configuration was used to produce this model:
base_model:
model:
path: meta-llama/Meta-Llama-3-8B
dtype: bfloat16
merge_method: dare_ties
parameters:
int8_mask: 1.0
normalize: 0.0
slices:
- sources:
- layer_range: [0, 32]
model:
model:
path: meta-llama/Meta-Llama-3-8B-Instruct
parameters:
density:
- filter: embed_token
value: 0.1312973174759711
- filter: model.norm
value: 0.9822573068643683
- filter: lm_head
value: 0.9822573068643683
- filter: layers.0.
value: 0.7585980531146381
- filter: layers.1.
value: 0.561884676687423
- filter: layers.2.
value: 0.9813332103529979
- filter: layers.3.
value: 0.21930890320643737
- filter: layers.4.
value: 0.8438595439416953
- filter: layers.5.
value: 0.7505829508553786
- filter: layers.6.
value: 0.25806652766624183
- filter: layers.7.
value: 0.09440963517101368
- filter: layers.8.
value: 0.7755855137764502
- filter: layers.9.
value: 0.980153628060705
- filter: layers.10.
value: 0.19005899180649255
- filter: layers.11.
value: 0.8607970083829785
- filter: layers.12.
value: 0.397155695730673
- filter: layers.13.
value: 0.13006504372382097
- filter: layers.14.
value: 0.09190339054787032
- filter: layers.15.
value: 0.14507208971010788
- filter: layers.16.
value: 0.8783567570968138
- filter: layers.17.
value: 0.12105889524706802
- filter: layers.18.
value: 0.8613122534393256
- filter: layers.19.
value: 0.04810799757747608
- filter: layers.20.
value: 0.5576780215726541
- filter: layers.21.
value: 0.027665563208104402
- filter: layers.22.
value: 0.7853866990645277
- filter: layers.23.
value: 0.8674754718436442
- filter: layers.24.
value: 0.915444005146838
- filter: layers.25.
value: 0.8408012482479134
- filter: layers.26.
value: 0.021388326989086454
- filter: layers.27.
value: 0.8382317625625855
- filter: layers.28.
value: 0.0418890647830244
- filter: layers.29.
value: 0.16347844552378538
- filter: layers.30.
value: 0.3152357116732385
- filter: layers.31.
value: 0.24560833074353305
weight:
- filter: embed_token
value: 0.07729925635324639
- filter: model.norm.
value: 0.8178953444711945
- filter: lm_head
value: 0.8178953444711945
- filter: layers.0.
value: 0.7744866238001816
- filter: layers.1.
value: 0.9434402042495054
- filter: layers.2.
value: 0.7277347873270702
- filter: layers.3.
value: 0.7344542377948238
- filter: layers.4.
value: 0.7789550773026275
- filter: layers.5.
value: 0.9977070050825179
- filter: layers.6.
value: 0.9790331299458723
- filter: layers.7.
value: 0.936694814017198
- filter: layers.8.
value: 0.8022245662093622
- filter: layers.9.
value: 0.9207818987772527
- filter: layers.10.
value: 0.9177346250309115
- filter: layers.11.
value: 0.9705518504378707
- filter: layers.12.
value: 0.6962192295710468
- filter: layers.13.
value: 0.6601913931204128
- filter: layers.14.
value: 0.7700302171860424
- filter: layers.15.
value: 0.8764914331693985
- filter: layers.16.
value: 0.1680474521036102
- filter: layers.17.
value: 0.7252502118522168
- filter: layers.18.
value: 0.974742254778171
- filter: layers.19.
value: 0.6195931122778885
- filter: layers.20.
value: 0.8808789662609544
- filter: layers.21.
value: 0.10257515634421632
- filter: layers.22.
value: 0.04739908873803297
- filter: layers.23.
value: 0.9609916339993614
- filter: layers.24.
value: 0.919873783591387
- filter: layers.25.
value: 0.8920927224651306
- filter: layers.26.
value: 0.3163445951176943
- filter: layers.27.
value: 0.9183620719063076
- filter: layers.28.
value: 0.7696116230696886
- filter: layers.29.
value: 0.8092141535971384
- filter: layers.30.
value: 0.9831761728133218
- filter: layers.31.
value: 0.8368550180785097
- layer_range: [0, 32]
model:
model:
path: meta-llama/Meta-Llama-3-8B
parameters:
density:
- filter: embed_token
value: 0.17966590577034688
- filter: model.norm
value: 0.21103453748756532
- filter: lm_head
value: 0.21103453748756532
- filter: layers.0.
value: 0.1258111935267289
- filter: layers.1.
value: 0.14712518700082466
- filter: layers.2.
value: 0.15119918492525058
- filter: layers.3.
value: 0.2786737466113946
- filter: layers.4.
value: 0.7941421047966343
- filter: layers.5.
value: 0.14996187005944175
- filter: layers.6.
value: 0.21464495779281623
- filter: layers.7.
value: 0.9544388237818261
- filter: layers.8.
value: 0.6823542810137563
- filter: layers.9.
value: 0.12285240879708
- filter: layers.10.
value: 0.9560979622228943
- filter: layers.11.
value: 0.18774045206190226
- filter: layers.12.
value: 0.040667440606417166
- filter: layers.13.
value: 0.06577308115912563
- filter: layers.14.
value: 0.10898066770591856
- filter: layers.15.
value: 0.7988665902205875
- filter: layers.16.
value: 0.6902648842556923
- filter: layers.17.
value: 0.21929644905849668
- filter: layers.18.
value: 0.031818815520317414
- filter: layers.19.
value: 0.09373597067550557
- filter: layers.20.
value: 0.05663841107031176
- filter: layers.21.
value: 0.573566646668988
- filter: layers.22.
value: 0.2100277505470118
- filter: layers.23.
value: 0.14741793214499152
- filter: layers.24.
value: 0.13421247708500247
- filter: layers.25.
value: 0.3923811614341299
- filter: layers.26.
value: 0.7128080834275775
- filter: layers.27.
value: 0.7173419548629656
- filter: layers.28.
value: 0.25148835909796347
- filter: layers.29.
value: 0.01041679056502079
- filter: layers.30.
value: 0.04502964289907793
- filter: layers.31.
value: 0.7933910983298705
weight:
- filter: embed_token
value: 0.1379551165348899
- filter: model.norm.
value: 0.1145495237372664
- filter: lm_head
value: 0.1145495237372664
- filter: layers.0.
value: 0.058088970620673676
- filter: layers.1.
value: 0.9812897914515345
- filter: layers.2.
value: 0.7981160133946364
- filter: layers.3.
value: 0.7634141926997863
- filter: layers.4.
value: 0.16065704042361334
- filter: layers.5.
value: 0.11985625529888592
- filter: layers.6.
value: 0.12337476562239155
- filter: layers.7.
value: 0.24780333206355964
- filter: layers.8.
value: 0.031049663247481193
- filter: layers.9.
value: 0.9399411230471199
- filter: layers.10.
value: 0.059148520666240975
- filter: layers.11.
value: 0.7231307420560733
- filter: layers.12.
value: 0.8604431238597543
- filter: layers.13.
value: 0.7874665970878372
- filter: layers.14.
value: 0.18061875009463604
- filter: layers.15.
value: 0.09518784175500823
- filter: layers.16.
value: 0.27798323894923355
- filter: layers.17.
value: 0.9413932817490506
- filter: layers.18.
value: 0.2512810899514295
- filter: layers.19.
value: 0.11709350712163674
- filter: layers.20.
value: 0.6739674929530162
- filter: layers.21.
value: 0.21208528089924455
- filter: layers.22.
value: 0.811536887881144
- filter: layers.23.
value: 0.6200821391879456
- filter: layers.24.
value: 0.7663192062414923
- filter: layers.25.
value: 0.8657672217068108
- filter: layers.26.
value: 0.10630047742154969
- filter: layers.27.
value: 0.9641883915469491
- filter: layers.28.
value: 0.9807033575417018
- filter: layers.29.
value: 0.04810198321663159
- filter: layers.30.
value: 0.8249574979326807
- filter: layers.31.
value: 0.01564957234872716