Martijn van Beers
commited on
Commit
•
66d5973
1
Parent(s):
265622b
Make the custom model work with IG
Browse filesIG doesn't create tensors that have `requires_grad`, so check for
that before adding the hook
- lib/roberta2.py +2 -1
lib/roberta2.py
CHANGED
@@ -280,7 +280,8 @@ class RobertaSelfAttention(nn.Module):
|
|
280 |
attention_probs = nn.functional.softmax(attention_scores, dim=-1)
|
281 |
|
282 |
self.save_attn(attention_probs)
|
283 |
-
attention_probs.
|
|
|
284 |
|
285 |
# This is actually dropping out entire tokens to attend to, which might
|
286 |
# seem a bit unusual, but is taken from the original Transformer paper.
|
|
|
280 |
attention_probs = nn.functional.softmax(attention_scores, dim=-1)
|
281 |
|
282 |
self.save_attn(attention_probs)
|
283 |
+
if attention_probs.requires_grad:
|
284 |
+
attention_probs.register_hook(self.save_attn_gradients)
|
285 |
|
286 |
# This is actually dropping out entire tokens to attend to, which might
|
287 |
# seem a bit unusual, but is taken from the original Transformer paper.
|