|
<!DOCTYPE html> |
|
<html lang="en"> |
|
|
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
<title>Training Commands</title> |
|
<style> |
|
body { |
|
font-family: monospace; |
|
margin: 0; |
|
padding: 8px; |
|
} |
|
|
|
#command { |
|
padding: 8px; |
|
background-color: #f6f8fa; |
|
border: 1px solid #d1d5da; |
|
border-radius: 3px; |
|
display: inline-block; |
|
white-space: pre-wrap; |
|
|
|
word-break: break-all; |
|
|
|
} |
|
|
|
button { |
|
margin-left: 8px; |
|
cursor: pointer; |
|
} |
|
</style> |
|
</head> |
|
|
|
<body> |
|
<div id="command">torchrun --nproc_per_node=8 --master_port=<YOUR PORT> train.py \ |
|
--model_name_or_path "facebook/opt-6.7b" \ |
|
--data_path medalpaca_small.json \ |
|
--bf16 True \ |
|
--output_dir models \ |
|
--num_train_epochs 3 \ |
|
--per_device_train_batch_size 4 \ |
|
--per_device_eval_batch_size 4 \ |
|
--gradient_accumulation_steps 8 \ |
|
--evaluation_strategy "no" \ |
|
--save_strategy "steps" \ |
|
--save_steps 2000 \ |
|
--save_total_limit 1 \ |
|
--learning_rate 2e-5 \ |
|
--weight_decay 0. \ |
|
--warmup_ratio 0.03 \ |
|
--lr_scheduler_type "cosine" \ |
|
--logging_steps 1 \ |
|
--fsdp "full_shard auto_wrap" \ |
|
--fsdp_transformer_layer_cls_to_wrap 'OPTDecoderLayer' \ |
|
--tf32 True</div> |
|
<div> |
|
<button onclick="changeCommand(0)">OPT 6.7B</button> |
|
<button onclick="changeCommand(0)">OPT 13B</button> |
|
<button onclick="changeCommand(1)">Alpaca 7B</button> |
|
</div> |
|
|
|
|
|
<script> |
|
const commands = [ |
|
`torchrun --nproc_per_node=8 --master_port=<YOUR PORT> train.py \\ |
|
--model_name_or_path "facebook/opt-6.7b" \\ |
|
--data_path medalpaca_small.json \\ |
|
--bf16 True \\ |
|
--output_dir models \\ |
|
--num_train_epochs 3 \\ |
|
--per_device_train_batch_size 4 \\ |
|
--per_device_eval_batch_size 4 \\ |
|
--gradient_accumulation_steps 8 \\ |
|
--evaluation_strategy "no" \\ |
|
--save_strategy "steps" \\ |
|
--save_steps 2000 \\ |
|
--save_total_limit 1 \\ |
|
--learning_rate 2e-5 \\ |
|
--weight_decay 0. \\ |
|
--warmup_ratio 0.03 \\ |
|
--lr_scheduler_type "cosine" \\ |
|
--logging_steps 1 \\ |
|
--fsdp "full_shard auto_wrap" \\ |
|
--fsdp_transformer_layer_cls_to_wrap 'OPTDecoderLayer' \\ |
|
--tf32 True`, |
|
|
|
`torchrun --nproc_per_node=8 --master_port=<YOUR PORT> train.py \\ |
|
--model_name_or_path "facebook/opt-13b" \\ |
|
--data_path medalpaca_small.json \\ |
|
--bf16 True \\ |
|
--output_dir models \\ |
|
--num_train_epochs 3 \\ |
|
--per_device_train_batch_size 2 \\ |
|
--per_device_eval_batch_size 2 \\ |
|
--gradient_accumulation_steps 16 \\ |
|
--evaluation_strategy "no" \\ |
|
--save_strategy "steps" \\ |
|
--save_steps 2000 \\ |
|
--save_total_limit 1 \\ |
|
--learning_rate 2e-5 \\ |
|
--weight_decay 0. \\ |
|
--warmup_ratio 0.03 \\ |
|
--lr_scheduler_type "cosine" \\ |
|
--logging_steps 1 \\ |
|
--fsdp "full_shard auto_wrap" \\ |
|
--fsdp_transformer_layer_cls_to_wrap 'OPTDecoderLayer' \\ |
|
--tf32 True`, |
|
|
|
`torchrun --nproc_per_node=8 --master_port=<YOUR PORT> train.py \\ |
|
--model_name_or_path <PATH_TO_LLAMA_WEIGHTS> \\ |
|
--data_path medalpaca_small.json \\ |
|
--bf16 True \\ |
|
--output_dir models \\ |
|
--num_train_epochs 3 \\ |
|
--per_device_train_batch_size 4 \\ |
|
--per_device_eval_batch_size 4 \\ |
|
--gradient_accumulation_steps 8 \\ |
|
--evaluation_strategy "no" \\ |
|
--save_strategy "steps" \\ |
|
--save_steps 2000 \\ |
|
--save_total_limit 1 \\ |
|
--learning_rate 2e-5 \\ |
|
--weight_decay 0. \\ |
|
--warmup_ratio 0.03 \\ |
|
--lr_scheduler_type "cosine" \\ |
|
--logging_steps 1 \\ |
|
--fsdp "full_shard auto_wrap" \\ |
|
--fsdp_transformer_layer_cls_to_wrap 'LlamaDecoderLayer' \\ |
|
--tf32 True`, |
|
]; |
|
|
|
function changeCommand(appIndex) { |
|
document.getElementById("command").innerText = commands[appIndex]; |
|
} |
|
</script> |
|
</body> |
|
|
|
</html> |
