Add appy_chat_template process

spanish_medica_llm.py

@@ -692,7 +692,7 @@ def run_training_process():
  
  train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
 
- base_model = loadBaseModel(MISTRAL_BASE_MODEL_ID)
+ base_model = loadBaseModel(MISTRAL_BASE_MODEL_ID)
  base_model = modelLoraConfigBioMistral(base_model)
 
  configAndRunTraining(base_model,train_dataset, eval_dataset, tokenizer)
@@ -703,6 +703,7 @@ def run_finnetuning_process():
  os.environ['WANDB_DISABLED'] = 'true'
  tokenizer = loadSpanishTokenizer()
  medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
+ print (medicalSpanishDataset[5])
  medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
  medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
  train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )