import gradio as gr
from openai import OpenAI
import os
# Retrieve the access token from the environment variable
ACCESS_TOKEN = os.getenv("HF_TOKEN")
print("Access token loaded.")
# Initialize the OpenAI client with the Hugging Face Inference API endpoint
client = OpenAI(
base_url="https://api-inference.huggingface.co/v1/",
api_key=ACCESS_TOKEN,
)
print("OpenAI client initialized.")
def respond(
message,
history: list[tuple[str, str]],
system_message,
custom_model,
model,
max_tokens,
temperature,
top_p,
frequency_penalty,
seed
):
"""
This function handles the chatbot response. It takes in:
- message: the user's new message
- history: the list of previous messages, each as a tuple (user_msg, assistant_msg)
- system_message: the system prompt
- custom_model: custom model path (if any)
- model: selected model from featured models
- max_tokens: the maximum number of tokens to generate in the response
- temperature: sampling temperature
- top_p: top-p (nucleus) sampling
- frequency_penalty: penalize repeated tokens in the output
- seed: a fixed seed for reproducibility; -1 will mean 'random'
"""
print(f"Received message: {message}")
print(f"History: {history}")
print(f"System message: {system_message}")
print(f"Custom model: {custom_model}")
print(f"Selected model: {model}")
print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}")
print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}")
# Convert seed to None if -1 (meaning random)
if seed == -1:
seed = None
# Construct the messages array required by the API
messages = [{"role": "system", "content": system_message}]
# Add conversation history to the context
for val in history:
user_part = val[0]
assistant_part = val[1]
if user_part:
messages.append({"role": "user", "content": user_part})
print(f"Added user message to context: {user_part}")
if assistant_part:
messages.append({"role": "assistant", "content": assistant_part})
print(f"Added assistant message to context: {assistant_part}")
# Append the latest user message
messages.append({"role": "user", "content": message})
# Start with an empty string to build the response as tokens stream in
response = ""
print("Sending request to OpenAI API.")
# Determine which model to use
if custom_model.strip():
selected_model = custom_model.strip()
else:
# Map the display names to actual model paths
model_mapping = {
"Llama 2 70B": "meta-llama/Llama-2-70b-chat-hf",
"Mixtral 8x7B": "mistralai/Mixtral-8x7B-Instruct-v0.1",
"Zephyr 7B": "HuggingFaceH4/zephyr-7b-beta",
"OpenChat 3.5": "openchat/openchat-3.5-0106",
}
selected_model = model_mapping.get(model, "meta-llama/Llama-2-70b-chat-hf")
# Make the streaming request to the HF Inference API via openai-like client
for message_chunk in client.chat.completions.create(
model=selected_model,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
frequency_penalty=frequency_penalty,
seed=seed,
messages=messages,
):
# Extract the token text from the response chunk
token_text = message_chunk.choices[0].delta.content
print(f"Received token: {token_text}")
response += token_text
yield response
print("Completed response generation.")
# Create a Chatbot component with a specified height
chatbot = gr.Chatbot(height=600)
print("Chatbot interface created.")
# Create the Gradio interface with tabs
with gr.Blocks(theme="Nymbo/Nymbo_Theme") as demo:
with gr.Row():
with gr.Column():
# Basic Settings Tab
with gr.Tab("Settings"):
# System Message
system_message = gr.Textbox(
value="",
label="System message",
placeholder="Enter a system message to guide the model's behavior"
)
# Model Selection Section
with gr.Accordion("Featured Models", open=True):
# Model Search
model_search = gr.Textbox(
label="Filter Models",
placeholder="Search for a featured model...",
lines=1
)
# Featured Models List
models_list = [
"Llama 2 70B",
"Mixtral 8x7B",
"Zephyr 7B",
"OpenChat 3.5"
]
model = gr.Radio(
label="Select a model",
choices=models_list,
value="Llama 2 70B"
)
# Custom Model Input
custom_model = gr.Textbox(
label="Custom Model",
info="Hugging Face model path (optional)",
placeholder="meta-llama/Llama-2-70b-chat-hf"
)
# Function to filter models
def filter_models(search_term):
filtered_models = [m for m in models_list if search_term.lower() in m.lower()]
return gr.update(choices=filtered_models)
# Update model list when search box is used
model_search.change(filter_models, inputs=model_search, outputs=model)
# Generation Parameters
with gr.Row():
max_tokens = gr.Slider(
minimum=1,
maximum=4096,
value=512,
step=1,
label="Max new tokens"
)
temperature = gr.Slider(
minimum=0.1,
maximum=4.0,
value=0.7,
step=0.1,
label="Temperature"
)
with gr.Row():
top_p = gr.Slider(
minimum=0.1,
maximum=1.0,
value=0.95,
step=0.05,
label="Top-P"
)
frequency_penalty = gr.Slider(
minimum=-2.0,
maximum=2.0,
value=0.0,
step=0.1,
label="Frequency Penalty"
)
with gr.Row():
seed = gr.Slider(
minimum=-1,
maximum=65535,
value=-1,
step=1,
label="Seed (-1 for random)"
)
# Information Tab
with gr.Tab("Information"):
# Featured Models Table
with gr.Accordion("Featured Models", open=True):
gr.HTML(
"""
"""
)
# Parameters Overview
with gr.Accordion("Parameters Overview", open=False):
gr.Markdown(
"""
## System Message
A message that sets the context and behavior for the model. This helps guide the model's responses.
## Max New Tokens
Controls the maximum length of the generated response. Higher values allow for longer outputs but may take more time.
## Temperature
Controls randomness in the output:
- Lower values (0.1-0.5): More focused and deterministic
- Higher values (0.7-1.0): More creative and diverse
- Very high values (>1.0): More random and potentially chaotic
## Top-P (Nucleus Sampling)
Controls the cumulative probability threshold for token selection:
- Lower values: More focused on highly likely tokens
- Higher values: Considers a wider range of possibilities
## Frequency Penalty
Adjusts the likelihood of token repetition:
- Negative values: May encourage repetition
- Zero: Neutral
- Positive values: Discourages repetition
## Seed
A number that controls the randomness in generation:
- -1: Random seed each time
- Fixed value: Reproducible outputs with same parameters
"""
)
# Set up the chat interface
chatbot = gr.Chatbot(height=600)
msg = gr.Textbox(label="Message")
clear = gr.ClearButton([msg, chatbot])
msg.submit(respond, [msg, chatbot, system_message, custom_model, model, max_tokens, temperature, top_p, frequency_penalty, seed], [chatbot, msg])
print("Launching the demo application.")
demo.launch(show_api=False, share=False)