d-Matrix commited on
Commit
0b572cf
1 Parent(s): b09fb1d

Update dmx_perplexity.py

Browse files

making sure seq_len is a multiple of stride

Files changed (1) hide show
  1. dmx_perplexity.py +1 -0
dmx_perplexity.py CHANGED
@@ -102,6 +102,7 @@ class DmxPerplexity(evaluate.Metric):
102
 
103
  stride = max_seq_len
104
  seq_len = encodings.input_ids.size(1)
 
105
 
106
  nlls = []
107
  prev_end_loc = 0
 
102
 
103
  stride = max_seq_len
104
  seq_len = encodings.input_ids.size(1)
105
+ seq_len = (seq_len // stride) * stride
106
 
107
  nlls = []
108
  prev_end_loc = 0