#!/bin/bash
CUDA_VISIBLE_DEVICES=0 python train.py  \
    --model_name_or_path tiiuae/falcon-7b \
    --bf16 True \
    --output_dir ./out_dir/  \
    --cache_dir ./hf-cache/ \
    --num_train_epochs 1  \
    --per_device_train_batch_size 1     \
    --per_device_eval_batch_size 1     \
    --gradient_accumulation_steps 1     \
    --evaluation_strategy "no"     \
    --save_strategy "steps"     \
    --save_steps 2000     \
    --save_total_limit 2     \
    --learning_rate 2e-5     \
    --weight_decay 0.1     \
    --warmup_ratio 0.03     \
    --lr_scheduler_type "cosine"     \
    --logging_steps 1     \
    --tf32 True \
    --max_steps 15000 \
    --model_max_length 1024 \
    --mem_freq 31