import gradio as gr from openai import OpenAI import os ACCESS_TOKEN = os.getenv("HF_TOKEN") print("Access token loaded.") client = OpenAI( base_url="https://api-inference.huggingface.co/v1/", api_key=ACCESS_TOKEN, ) print("OpenAI client initialized.") def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, frequency_penalty, seed, custom_model ): print(f"Received message: {message}") print(f"History: {history}") print(f"System message: {system_message}") if seed == -1: seed = None messages = [{"role": "system", "content": system_message}] for val in history: if val[0]: messages.append({"role": "user", "content": val[0]}) if val[1]: messages.append({"role": "assistant", "content": val[1]}) messages.append({"role": "user", "content": message}) model_to_use = custom_model.strip() if custom_model.strip() != "" else "Qwen/Qwen2.5-72B-Instruct" response = "" for message_chunk in client.chat.completions.create( model=model_to_use, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p, frequency_penalty=frequency_penalty, seed=seed, messages=messages, ): token_text = message_chunk.choices[0].delta.content response += token_text yield response chatbot = gr.Chatbot(height=600, show_copy_button=True, placeholder="ChatGPT is initializing...", likeable=True, layout="panel") system_message_box = gr.Label(value="You can select Max Tokens, Temperature, Top-P, Seed") max_tokens_slider = gr.Slider(1024, 2048, value=1024, step=100, label="Max new tokens") temperature_slider = gr.Slider(0.1, 1.0, value=0.7, step=0.1, label="Temperature") top_p_slider = gr.Slider(0.1, 1.0, value=0.95, step=0.05, label="Top-P") frequency_penalty_slider = gr.Slider(-2.0, 2.0, value=0.0, step=0.1, label="Frequency Penalty") seed_slider = gr.Slider(-1, 65535, value=-1, step=1, label="Seed (-1 for random)") custom_model_box = gr.Textbox(value="Qwen/Qwen2.5-72B-Instruct", label="AI Mode is ") demo = gr.ChatInterface( fn=respond, additional_inputs=[ system_message_box, max_tokens_slider, temperature_slider, top_p_slider, frequency_penalty_slider, seed_slider, custom_model_box, ], fill_height=True, chatbot=chatbot, theme="Nymbo/Nymbo_Theme", ) if __name__ == "__main__": print("Launching the ChatGPT-Llama...") demo.launch()