Spaces:
Runtime error
Runtime error
Add appy_chat_template process
Browse files- spanish_medica_llm.py +2 -1
spanish_medica_llm.py
CHANGED
@@ -703,7 +703,8 @@ def run_finnetuning_process():
|
|
703 |
os.environ['WANDB_DISABLED'] = 'true'
|
704 |
tokenizer = loadSpanishTokenizer()
|
705 |
medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
|
706 |
-
print (medicalSpanishDataset[5])
|
|
|
707 |
medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
|
708 |
medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
|
709 |
train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
|
|
|
703 |
os.environ['WANDB_DISABLED'] = 'true'
|
704 |
tokenizer = loadSpanishTokenizer()
|
705 |
medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
|
706 |
+
print ( tokenizer.apply_chat_template(medicalSpanishDataset[5], tokenize=False))
|
707 |
+
print('----------------------------------------------------------')
|
708 |
medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
|
709 |
medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
|
710 |
train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
|