|
import transformers |
|
import torch |
|
|
|
|
|
model_path_name = "SicariusSicariiStuff/LLAMA-3_8B_Unaligned_BETA" |
|
|
|
|
|
pipeline = transformers.pipeline( |
|
"text-generation", |
|
model=model_path_name, |
|
model_kwargs={"torch_dtype": torch.bfloat16}, |
|
device_map="auto", |
|
) |
|
|
|
|
|
message_list = [ |
|
[ |
|
{'role': 'system', 'content': "You are an AI assistant."}, |
|
{'role': 'user', 'content': "Who are you?"} |
|
] |
|
] |
|
|
|
|
|
try: |
|
prompts = [ |
|
pipeline.tokenizer.apply_chat_template( |
|
messages, |
|
tokenize=False, |
|
add_generation_prompt=True, |
|
) |
|
for messages in message_list |
|
] |
|
except AttributeError: |
|
|
|
prompts = [ |
|
f"<|im_start|>system\n{msg[0]['content']}<|im_end|>\n" |
|
f"<|im_start|>user\n{msg[1]['content']}<|im_end|>\n<|im_start|>assistant\n" |
|
for msg in message_list |
|
] |
|
|
|
|
|
print("Formatted Prompts:", prompts) |
|
|
|
|
|
eos_token_id = pipeline.tokenizer.eos_token_id or 50256 |
|
pad_token_id = eos_token_id |
|
print("EOS Token ID:", eos_token_id) |
|
|
|
|
|
tokens = pipeline.tokenizer(prompts, padding=True, return_tensors="pt") |
|
print("Tokenized Input:", tokens) |
|
|
|
|
|
try: |
|
outputs = pipeline( |
|
prompts, |
|
max_new_tokens=100, |
|
do_sample=True, |
|
temperature=0.5, |
|
top_p=0.5, |
|
eos_token_id=eos_token_id, |
|
pad_token_id=pad_token_id, |
|
) |
|
print("Outputs:", outputs) |
|
except Exception as e: |
|
print("Error during generation:", str(e)) |
|
|
|
|
|
|