fix calculation warm up step to ratio

modeling5.py

@@ -7,7 +7,7 @@ class MiniSunConfig:
     def __init__(self, vocab_size=30522, max_position_embeddings=1024, hidden_size=512,
                  num_attention_heads=8, intermediate_size=2048, num_hidden_layers=8,
                  dropout_rate=0.1, weight_decay=0.01, learning_rate=1e-4, total_steps=2500,
-                 warmup_steps=500, restart_period=500):
+                 warmup_ratio=0.5, restart_period=500):
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.hidden_size = hidden_size
@@ -18,7 +18,7 @@ class MiniSunConfig:
         self.weight_decay = weight_decay
         self.learning_rate = learning_rate
         self.total_steps = total_steps
-        self.warmup_steps = warmup_steps
+        self.warmup_ratio = warmup_ratio
         self.restart_period = restart_period
 
 
@@ -140,7 +140,7 @@ def create_model(config):
 
 def cosine_annealing_with_warmup(step, config):
     """Learning rate schedule with warm-up and cosine annealing."""
-    warmup_steps = int(config.total_steps * config.warmup_steps)
+    warmup_steps = int(config.total_steps * config.warmup_ratio)
     if step < warmup_steps:
         return config.learning_rate * (step / warmup_steps)
     else:
@@ -150,7 +150,7 @@ def cosine_annealing_with_warmup(step, config):
 
 def cosine_annealing_with_restarts(step, config):
     """Learning rate schedule with warm-up and cosine annealing with restarts."""
-    warmup_steps = int(config.total_steps * config.warmup_steps)
+    warmup_steps = int(config.total_steps * config.warmup_ratio)
 
     current_cycle = step // config.restart_period
     effective_step = step % config.restart_period