app.py

import os
import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype="float16",
    bnb_4bit_use_double_quant=False,
)
device_map = {"": 0}

model_name = "rukaiyah-indika-ai/rv-chatbot-2"

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config
)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
from transformers import pipeline
inst = "You are a very helpful assistant providing solutions to road-related queries. Ensure you provide correct and relevant answers. If you don't know the answer to a question, please don't share false information. "
prompt = "Hello"

pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, temperature=0.2, max_new_tokens=256)
ranked_results = pipe(f"<s>[INST] {inst}{prompt} [/INST]")
for result in ranked_results:
    response = result['generated_text']
    response = response.split("[/INST]", 1)[-1]
    response = response.replace("<s>", "")
import gradio as gr

def generate_response(prompt):
    inst = "You are a very helpful assistant providing solutions to road-related queries. Ensure you provide correct and relevant answers according to the IRC guidelines. If you don't know the answer to a question, please don't share false information."
    pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, temperature=0.2, max_new_tokens=256)
    ranked_results = pipe(f"<s>[INST] {inst}{prompt} [/INST]")

    for result in ranked_results:
        response = result['generated_text']
        response = response.split("[/INST]", 1)[-1]
        response = response.replace("<s>", "")
        response = response.replace("</s>", "")
        return response

iface = gr.Interface(
    fn=generate_response,
    inputs="text",
    outputs="text",
    title="Road-GPT",
    description="Enter your query related to road management and get a response generated by an AI model."
)

iface.launch(share=True)