Update fine-tune-modernbert-classifier.ipynb
Browse files
examples/fine-tune-modernbert-classifier.ipynb
CHANGED
@@ -178,7 +178,7 @@
|
|
178 |
" \n",
|
179 |
"# Tokenize helper function\n",
|
180 |
"def tokenize(batch):\n",
|
181 |
-
" return tokenizer(batch['text'], padding=
|
182 |
" \n",
|
183 |
"# Tokenize dataset\n",
|
184 |
"if \"label\" in split_dataset[\"train\"].features.keys():\n",
|
|
|
178 |
" \n",
|
179 |
"# Tokenize helper function\n",
|
180 |
"def tokenize(batch):\n",
|
181 |
+
" return tokenizer(batch['text'], padding=True, truncation=True, return_tensors=\"pt\")\n",
|
182 |
" \n",
|
183 |
"# Tokenize dataset\n",
|
184 |
"if \"label\" in split_dataset[\"train\"].features.keys():\n",
|