Kevin Fink
commited on
Commit
·
75c24c0
1
Parent(s):
5a28d7d
init
Browse files
app.py
CHANGED
|
@@ -77,7 +77,7 @@ def fine_tune_model(model_name, dataset_name, hub_id, api_key, num_epochs, batch
|
|
| 77 |
return tokenized_datasets
|
| 78 |
|
| 79 |
# Tokenize the dataset in chunks
|
| 80 |
-
tokenized_datasets = tokenize_in_chunks(dataset
|
| 81 |
|
| 82 |
# If you want to combine all chunks into a single dataset
|
| 83 |
final_tokenized_dataset = concatenate_datasets(tokenized_datasets)
|
|
|
|
| 77 |
return tokenized_datasets
|
| 78 |
|
| 79 |
# Tokenize the dataset in chunks
|
| 80 |
+
tokenized_datasets = tokenize_in_chunks(dataset, chunk_size=1000)
|
| 81 |
|
| 82 |
# If you want to combine all chunks into a single dataset
|
| 83 |
final_tokenized_dataset = concatenate_datasets(tokenized_datasets)
|