import spaces import json import subprocess from llama_cpp import Llama from llama_cpp_agent import LlamaCppAgent from llama_cpp_agent.providers import LlamaCppPythonProvider from llama_cpp_agent.chat_history import BasicChatHistory from llama_cpp_agent.chat_history.messages import Roles import gradio as gr from huggingface_hub import hf_hub_download hf_hub_download( repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", filename="dorna-llama3-8b-instruct.Q4_0.gguf", local_dir = "." ) hf_hub_download( repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", filename="dorna-llama3-8b-instruct.Q8_0.gguf", local_dir = "." ) hf_hub_download( repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", filename="dorna-llama3-8b-instruct.Q4_0.gguf", local_dir = "." ) hf_hub_download( repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", filename="dorna-llama3-8b-instruct.Q5_0.gguf", local_dir = "." ) hf_hub_download( repo_id="PartAI/Dorna-Llama3-8B-Instruct-GGUF", filename="dorna-llama3-8b-instruct.bf16.gguf", local_dir = "." ) css = """ .message-row { justify-content: space-evenly !important; } .message-bubble-border { border-radius: 6px !important; } .dark.message-bubble-border { border-color: #343140 !important; } .dark.user { background: #1e1c26 !important; } .dark.assistant.dark, .dark.pending.dark { background: #16141c !important; } @import url('https://fonts.googleapis.com/css2?family=Vazirmatn&display=swap'); body, .gradio-container, .gr-button, .gr-input, .gr-slider, .gr-dropdown, .gr-markdown { font-family: 'Vazirmatn', sans-serif !important; } ._button { font-size: 20px; } pre, code { direction: ltr !important; unicode-bidi: plaintext !important; } """ def get_messages_formatter_type(model_name): from llama_cpp_agent import MessagesFormatterType return MessagesFormatterType.CHATML @spaces.GPU(duration=120) def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, top_k, repeat_penalty, model, ): chat_template = get_messages_formatter_type(model) llm = Llama( model_path=f"./{model}", flash_attn=True, n_threads=40, n_gpu_layers=81, n_batch=1024, n_ctx=8192, ) provider = LlamaCppPythonProvider(llm) agent = LlamaCppAgent( provider, system_prompt=f"{system_message}", predefined_messages_formatter_type=chat_template, debug_output=True ) settings = provider.get_provider_default_settings() settings.temperature = temperature settings.top_k = top_k settings.top_p = top_p settings.max_tokens = max_tokens settings.repeat_penalty = repeat_penalty settings.stream = True messages = BasicChatHistory() for msn in history: user = { 'role': Roles.user, 'content': msn[0] } assistant = { 'role': Roles.assistant, 'content': msn[1] } messages.add_message(user) messages.add_message(assistant) stream = agent.get_chat_response( message[-2:], llm_sampling_settings=settings, chat_history=messages, returns_streaming_generator=True, print_output=False ) outputs = "" for output in stream: outputs += output yield outputs PLACEHOLDER = """

با فرمت‌های GGUF درنا چت کنید!

""" demo = gr.ChatInterface( respond, additional_inputs=[ gr.Textbox(value="You are a helpful Persian assistant. Please answer questions in the asked language.", label="System message", rtl=False), gr.Dropdown([ 'dorna-llama3-8b-instruct.Q8_0.gguf', 'dorna-llama3-8b-instruct.Q4_0.gguf', 'dorna-llama3-8b-instruct.Q5_0.gguf', 'dorna-llama3-8b-instruct.bf16.gguf', ], value="dorna-llama3-8b-instruct.Q8_0.gguf", label="Model" ), gr.Slider(minimum=1, maximum=8192, value=2048, step=1, label="Max tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.85, step=0.05, label="Top-p", ), gr.Slider( minimum=0, maximum=100, value=40, step=1, label="Top-k", ), gr.Slider( minimum=0.0, maximum=2.0, value=1, step=0.1, label="Repetition penalty", ), ], theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( body_background_fill_dark="#16141c", block_background_fill_dark="#16141c", block_border_width="1px", block_title_background_fill_dark="#1e1c26", input_background_fill_dark="#292733", button_secondary_background_fill_dark="#24212b", border_color_primary_dark="#343140", background_fill_secondary_dark="#16141c", color_accent_soft_dark="transparent" ), css=css, retry_btn="🔄 تلاش مجدد", undo_btn="↩️ بازگشت", clear_btn="🗑️ پاک کردن", submit_btn="ارسال", title="درنا، محصول مرکز تحقیقات هوش مصنوعی پارت", textbox=gr.Textbox(show_label=False, lines=2, rtl=True, placeholder="ورودی", show_copy_button=True, scale=4), chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER, rtl=True) ) if __name__ == "__main__": demo.launch()