config.yaml

exp_name: 'cosine_lr'

# Training dataset (from Huggingface)
data_source: "MedCat/MedCAT-PT-v1"

# The base model (from HuggingFace model hub)
model_name: "SeaLLMs/SeaLLMs-v3-1.5B"

# Tokenizer
tokenizer_device: 'cpu' # 'cpu', 'cuda:0', 'cuda:1'
tokenizer_batch_size: 1_000
max_length: 512

# Checkpoints configuration
output_folder: "./checkpoints/MedCAT-PT"  # Where to save checkpoints during the training
save_total_limit: 5  # Limit on number of checkpoints to keep
save_model_to: "./checkpoints/MedCAT-PT/"  # Where to save the last checkpoint + base_model + data_version
save_strategy: "steps"  # Saving strategy (either 'steps' or 'epoch')
save_steps: 10_000  # Save model every ... steps

# Logging configuration
logging_dir: "./logs"  # Directory for logs + base_model + data_version
logging_steps: 100  # Frequency of logging

# Training configuration
learning_rate: 5e-5 # Default 5e-5
lr_scheduler_type: cosine # default linear
warmup_steps: 2000 # default 0

per_device_train_batch_size: 4  # Training batch size
per_device_eval_batch_size: 4  # Evaluation batch size
num_train_epochs: 1  # Number of epochs
# max_steps: 500  # Total training steps (or use num_train_epochs instead)
eval_steps: 10_000  # Frequency of evaluation. Should equal to logging_steps (can be different, but should be equal)
evaluation_samples: 20_000 # evaluation samples used to evaluate the model during training process
evaluation_strategy: "steps"  # Evaluation strategy (either 'steps' or 'epoch')
seed: 3407  # Random seed for reproducibility