Update modeling_bert.py

modeling_bert.py

@@ -384,7 +384,8 @@ class BertSelfAttention(nn.Module):
             attention_scores = attention_scores + attention_mask
 
         # Normalize the attention scores to probabilities.
-        attention_probs = nn.functional.softmax(attention_scores, dim=-1)
+        #attention_probs = nn.functional.softmax(attention_scores, dim=-1)
+        attention_probs = softmax_1(attention_scores, dim=-1)
 
         # This is actually dropping out entire tokens to attend to, which might
         # seem a bit unusual, but is taken from the original Transformer paper.