Spaces:
Sleeping
Sleeping
d-Matrix
commited on
Commit
•
0b572cf
1
Parent(s):
b09fb1d
Update dmx_perplexity.py
Browse filesmaking sure seq_len is a multiple of stride
- dmx_perplexity.py +1 -0
dmx_perplexity.py
CHANGED
@@ -102,6 +102,7 @@ class DmxPerplexity(evaluate.Metric):
|
|
102 |
|
103 |
stride = max_seq_len
|
104 |
seq_len = encodings.input_ids.size(1)
|
|
|
105 |
|
106 |
nlls = []
|
107 |
prev_end_loc = 0
|
|
|
102 |
|
103 |
stride = max_seq_len
|
104 |
seq_len = encodings.input_ids.size(1)
|
105 |
+
seq_len = (seq_len // stride) * stride
|
106 |
|
107 |
nlls = []
|
108 |
prev_end_loc = 0
|