Fix generate process

spanish_medica_llm.py

@@ -25,7 +25,7 @@ from transformers import (
 
 from accelerate import FullyShardedDataParallelPlugin, Accelerator
 from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
-from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from peft import PeftConfig, LoraConfig, get_peft_model, prepare_model_for_kbit_training
 #import wandb
 from trl import SFTTrainer
 
@@ -735,8 +735,11 @@ def generate_response(query):
  top_k=40
  num_beams=2
 
- tokenizer = loadSpanishTokenizer()
- model = loadBaseModel(HUB_MODEL_ID)
+ #model = loadBaseModel(HUB_MODEL_ID)
+
+ config = PeftConfig.from_pretrained(HUB_MODEL_ID)
+ model = AutoModelForCausalLM.from_pretrained(HUB_MODEL_ID, return_dict=True, load_in_8bit=True, device_map={"":0})
+ tokenizer = AutoTokenizer.from_pretrained(HUB_MODEL_ID)
 
  #system = f"[INST]\nYou are a helpful coding assistant.[/INST]\n"
  query = get_chat_format({'raw_text':query, 'topic':''})