trainer.json

{
  "repo_id": "habanoz/haber-gpt-3-40M-8k-c4tr-v1.003",
  "trainer_config": {
    "seed": 145,
    "seq_length": 1024,
    "gradient_accumulation_steps": 6,
    "batch_size": 40,
    "data_dir": "c4tr.003",
    "max_iters": 20000,
    "warmup_iters": 2000,
    "grad_norm_clip": 1.0,
    "out_dir": "haber-gpt-3-40M-8k-c4tr-v1.003",
    "dtype": "float16",
    "compile": true,
    "gc": false,
    "learning_rate": 0.0018,
    "decay_lr": true,
    "lr_decay_iters": 20000,
    "min_lr": 0.00018,
    "weight_decay": 0.1,
    "beta1": 0.9,
    "beta2": 0.95,
    "log_interval": 50,
    "eval_interval": 1000,
    "eval_iters": 100,
    "promised_flops": 65000000000000.0,
    "wandb_log": true,
    "wandb_project": "Haber-GPT-3-40M",
    "wandb_run_name": "haber-gpt-3-40M-8k-c4tr-v1.003",
    "wandb_run_id": "1733943696"
  }
}