File size: 1,485 Bytes
35678b2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
model_name: Llama-3.2-1B
model_config:
name: ''
hf_config: {}
scale_embeddings: false
block_size: 8192
vocab_size: 32768
padding_multiple: 512
padded_vocab_size: 32768
n_layer: 8
n_head: 32
head_size: 64
n_embd: 512
rotary_percentage: 1.0
parallel_residual: true
bias: false
lm_head_bias: false
n_query_groups: 8
shared_attention_norm: false
norm_class_name: RMSNorm
post_attention_norm: false
post_mlp_norm: false
norm_eps: 1.0e-05
mlp_class_name: LLaMAMLP
gelu_approximate: none
intermediate_size: 2048
rope_condense_ratio: 1
rope_base: 500000
rope_adjustments:
factor: 32.0
low_freq_factor: 1.0
high_freq_factor: 4.0
original_max_seq_len: 8192
n_expert: 0
n_expert_per_token: 0
out_dir: ../out/contrain
precision: bf16-true
initial_checkpoint_dir: ../out/pretrain_checkpoint/final
resume: false
data:
class_path: litgpt.data.LitData
init_args:
data_path: ../contrain-data/
seed: 42
num_workers: 32
train:
save_interval: 200
log_interval: 1
global_batch_size: 512
micro_batch_size: 4
lr_warmup_steps: 1000
max_tokens: 9813468891
tie_embeddings: true
max_norm: 1.0
min_lr: 1.0e-05
eval:
interval: 100
max_iters: 100
initial_validation: false
final_validation: true
optimizer:
class_path: grokadamw.GrokAdamW
init_args:
lr: 4.0e-05
weight_decay: 0.01
betas:
- 0.9
- 0.999
devices: auto
num_nodes: 1
tokenizer_dir: ..
logger_name: wandb
seed: 23
|