pretrain model
Browse files
scripts/prepare_pretrain_dataset.py
CHANGED
|
@@ -94,7 +94,7 @@ datasets_configs = [
|
|
| 94 |
*[
|
| 95 |
{'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['text']}
|
| 96 |
for i in range(0, 100, 5)
|
| 97 |
-
]
|
| 98 |
|
| 99 |
#
|
| 100 |
# misc
|
|
|
|
| 94 |
*[
|
| 95 |
{'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 5}%]', 'format': lambda n: n['text']}
|
| 96 |
for i in range(0, 100, 5)
|
| 97 |
+
],
|
| 98 |
|
| 99 |
#
|
| 100 |
# misc
|