Tonic commited on
Commit
cfcb9b3
1 Parent(s): 8e93464

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -10
app.py CHANGED
@@ -13,21 +13,16 @@ examplecofee = """A cortado is a Spanish beverage consisting of espresso mixed w
13
 
14
  model_path = "NousResearch/Genstruct-7B"
15
 
16
- # hf_token = os.getenv("HF_TOKEN")
17
- # if not hf_token:
18
- # raise ValueError("Hugging Face token not found. Please set the HF_TOKEN environment variable.")
19
-
20
  tokenizer = AutoTokenizer.from_pretrained(model_path)
21
- # quantization_config = BitsAndBytesConfig(load_in_8bit=True)
22
- model = AutoModelForCausalLM.from_pretrained( model_path, device_map='cuda', load_in_8bit=True)
23
 
24
  @spaces.GPU
25
  def generate_text(prompt, custom_prompt, temperature, max_length):
26
- structured_input = f"[Title]: {prompt}\n[Content]: {custom_prompt}"
27
- inputs = tokenizer(structured_input, return_tensors='pt').to('cuda')
28
- generated_ids = model.generate(**inputs, max_new_tokens=max_length, temperature=temperature, do_sample=True).split(tokenizer.eos_token)[0])
29
  generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
30
-
31
  return generated_text
32
 
33
 
 
13
 
14
  model_path = "NousResearch/Genstruct-7B"
15
 
 
 
 
 
16
  tokenizer = AutoTokenizer.from_pretrained(model_path)
17
+ quantization_config = BitsAndBytesConfig(load_in_8bit=True)
18
+ model = AutoModelForCausalLM.from_pretrained( model_path, device_map='cuda', quantization_config=quantization_config)
19
 
20
  @spaces.GPU
21
  def generate_text(prompt, custom_prompt, temperature, max_length):
22
+ structured_input = f"[[[Title]]]: {prompt}\n[[[Content]]]: {custom_prompt}"
23
+ inputs = tokenizer(structured_input, return_tensors='pt').to('cuda')
24
+ generated_ids = model.generate(**inputs, max_new_tokens=max_length, temperature=temperature, pad_token_id=tokenizer.eos_token_id)
25
  generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
 
26
  return generated_text
27
 
28