DNA-LLM/synth_model_medium_v2

README.md

@@ -1,7 +1,6 @@
 ---
 library_name: transformers
-license: apache-2.0
-base_model: DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce
+base_model: DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy
 tags:
 - generated_from_trainer
 model-index:
@@ -14,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # virus-pythia-1.2M-2048-ce_synthetic
 
-This model is a fine-tuned version of [DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce](https://huggingface.co/DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce) on an unknown dataset.
+This model is a fine-tuned version of [DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy](https://huggingface.co/DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy) on an unknown dataset.
 
 ## Model description
 
@@ -34,19 +33,23 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-05
-- train_batch_size: 16
-- eval_batch_size: 8
+- train_batch_size: 480
+- eval_batch_size: 4
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 64
+- total_train_batch_size: 1920
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 1
 - num_epochs: 1
 
+### Training results
+
+
+
 ### Framework versions
 
-- Transformers 4.45.1
-- Pytorch 2.4.0
+- Transformers 4.45.2
+- Pytorch 2.4.1+cu121
 - Datasets 3.0.1
-- Tokenizers 0.20.0
+- Tokenizers 0.20.1

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce",
+  "_name_or_path": "DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy",
   "architectures": [
     "GPTNeoXForCausalLM"
   ],
@@ -10,14 +10,14 @@
   "eos_token_id": 0,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
-  "hidden_size": 768,
+  "hidden_size": 128,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 512,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 2048,
   "model_type": "gpt_neox",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
   "partial_rotary_factor": 0.25,
   "rope_scaling": null,
   "rope_theta": 10000,
@@ -25,7 +25,7 @@
   "rotary_pct": 0.25,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.45.1",
+  "transformers_version": "4.45.2",
   "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 1867

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 0,
   "eos_token_id": 0,
-  "transformers_version": "4.45.1"
+  "transformers_version": "4.45.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7c731a8a5c96581d50752424002548eb6fa8fe232626f5595fcb598baef8a5a
-size 351711712
+oid sha256:96717780fbae26014418efece25e3f70f2dc13855c69b2fae50835fc78b60de8
+size 6679704

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d38ac71a12b28c59d500c155cfbde41e0dbeae3a7a6c1fdb6731ddd928de37b
+oid sha256:7c4575a22107a1146a8cc4ab2f80f2f84d8531feb64d908608cde0bfcdfd8118
 size 5176