File size: 2,451 Bytes
8ccde71 edee460 8ccde71 99ba7cf 8ccde71 2494cf8 8ccde71 539a79c c6a0483 539a79c 633ccb5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
---
base_model:
- Qwen/Qwen2.5-3B-Instruct
tags:
- text-generation-inference
- transformers
- qwen2
- trl
- grpo
license: apache-2.0
language:
- en
---
# Uploaded model
- **Developed by:** TethysAI
- **License:** apache-2.0
- **Finetuned from model :** Qwen/Qwen2.5-3B-Instruct
# Follow the below structure to call the model:
```python
from transformers import AutoTokenizer, AutoModelForCausalLM
import torch
# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained("saishshinde15/TethysAI_Base_Reasoning")
model = AutoModelForCausalLM.from_pretrained("saishshinde15/TethysAI_Base_Reasoning")
# Prepare input prompt using chat template
SYSTEM_PROMPT = """
Respond in the following format:
<reasoning>
...
</reasoning>
<answer>
...
</answer>
"""
text = tokenizer.apply_chat_template([
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": "What is 2x+3=4"},
], tokenize=False, add_generation_prompt=True)
# Tokenize input
input_ids = tokenizer(text, return_tensors="pt").input_ids
# Move to GPU if available
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)
input_ids = input_ids.to(device)
# Generate response
# The line below caused the error as the loaded model doesn't have the attribute 'fast_generate'
# output_ids = model.generate(
# input_ids,
# temperature=0.8,
# top_p=0.95,
# max_length=1024, # Equivalent to max_tokens
# )
# Instead, use this
from vllm import SamplingParams
sampling_params = SamplingParams(
temperature=0.8,
top_p=0.95,
max_tokens=1024,
)
output = model.generate(
input_ids,
sampling_params=sampling_params,
)
# Decode and print output
output_text = tokenizer.decode(output[0], skip_special_tokens=True)
print(output_text)
```
<details>
<summary>Fast inference</summary>
```python
pip install transformers vllm vllm[lora] torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
text = tokenizer.apply_chat_template([
{"role" : "system", "content" : SYSTEM_PROMPT},
{"role" : "user", "content" : "What is 2x+3=4"},
], tokenize = False, add_generation_prompt = True)
from vllm import SamplingParams
sampling_params = SamplingParams(
temperature = 0.8,
top_p = 0.95,
max_tokens = 1024,
)
output = model.fast_generate(
text,
sampling_params = sampling_params,
lora_request = model.load_lora("grpo_saved_lora"),
)[0].outputs[0].text
output
```
</details> |