llama2_response_mail_generator.py

from huggingface_hub import hf_hub_download

from llama_cpp import Llama

model_name_or_path = "TheBloke/Llama-2-13B-chat-GGML"
model_basename = "llama-2-13b-chat.ggmlv3.q5_1.bin" # The model is in bin format

# Download the model file
model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)

# Initialize the Llama model with appropriate settings for GPU
lcpp_llm = Llama(
 model_path=model_path,
 n_threads=2, # CPU cores to use
 n_batch=512, # Batch size for processing; adjust as per your VRAM capacity
 n_gpu_layers=32 # Number of layers to run on GPU, dependent on your GPU's VRAM
)

def generate_email_response(email_prompt):
 # Check input received by the function
 print("Received prompt:", email_prompt)

 # Determine if the input is a shorthand command or an actual email
 if 'email to' in email_prompt.lower():
 # Assume it's a shorthand command, format appropriately
 formatted_prompt = f'''
 Email received: "{email_prompt}"
 Respond to this email, ensuring a professional tone, providing a concise update, and addressing any potential concerns the sender might have.
 Response:
 '''
 else:
 # Assume it's direct email content
 formatted_prompt = f'''
 Email received: "{email_prompt}"
 Respond to this email, ensuring a professional tone, providing a concise update, and addressing any potential concerns the sender might have.
 Response:
 '''

 # Generate response using Llama-2 model
 try:
 response = lcpp_llm(
 prompt=formatted_prompt,
 max_tokens=256,
 temperature=0.5,
 top_p=0.95,
 repeat_penalty=1.2,
 top_k=150,
 echo=True
 )
 generated_response = response["choices"][0]["text"]
 # Remove the input part from the output if it is included
 if formatted_prompt in generated_response:
 generated_response = generated_response.replace(formatted_prompt, '').strip()
 print("Generated response:", generated_response)
 return generated_response
 except Exception as e:
 print("Error in response generation:", str(e))
 return "Failed to generate response, please check the console for errors."