Upload InternLM2ForCausalLM

modeling_internlm2.py

@@ -390,7 +390,7 @@ class InternLM2Attention(nn.Module):
         #             attn_weights_i = attn_weights_i + causal_mask_i
 
         #         # Upcast attention to fp32 and apply softmax
-        #         attn_weights_i = F.softmax(attn_weights_i, dim=-1, dtype=torch.float32).to(query_states_i.dtype)
+        #         attn_weights_i = nn.functional.softmax(attn_weights_i, dim=-1, dtype=torch.float32).to(query_states_i.dtype)
         #         attn_output_i = torch.matmul(attn_weights_i, value_states.to(device))
                 
         #         return attn_output_i