|
import uvicorn |
|
from fastapi import FastAPI, HTTPException, Request |
|
from fastapi.middleware.cors import CORSMiddleware |
|
from fastapi.responses import HTMLResponse, StreamingResponse |
|
from huggingface_hub import InferenceClient |
|
from pydantic import BaseModel |
|
|
|
app = FastAPI() |
|
|
|
|
|
app.add_middleware( |
|
CORSMiddleware, |
|
allow_origins=["*"], |
|
allow_methods=["*"], |
|
allow_headers=["*"], |
|
) |
|
|
|
client = InferenceClient("Qwen/Qwen2.5-Coder-32B-Instruct") |
|
|
|
class ChatRequest(BaseModel): |
|
message: str |
|
history: list[tuple[str, str]] |
|
system_message: str |
|
max_tokens: int |
|
temperature: float |
|
top_p: float |
|
|
|
def generate_response(messages, max_tokens, temperature, top_p): |
|
for chunk in client.chat_completion( |
|
messages, |
|
max_tokens=max_tokens, |
|
stream=True, |
|
temperature=temperature, |
|
top_p=top_p, |
|
): |
|
yield chunk.choices[0].delta.content or "" |
|
|
|
@app.post("/api/chat") |
|
async def chat_stream(request: ChatRequest): |
|
try: |
|
messages = [{"role": "system", "content": request.system_message}] |
|
|
|
for user_msg, assistant_msg in request.history: |
|
messages.extend([ |
|
{"role": "user", "content": user_msg}, |
|
{"role": "assistant", "content": assistant_msg} |
|
]) |
|
|
|
messages.append({"role": "user", "content": request.message}) |
|
|
|
return StreamingResponse( |
|
generate_response( |
|
messages=messages, |
|
max_tokens=request.max_tokens, |
|
temperature=request.temperature, |
|
top_p=request.top_p |
|
), |
|
media_type="text/event-stream" |
|
) |
|
except Exception as e: |
|
raise HTTPException(status_code=500, detail=str(e)) |
|
|
|
@app.get("/", response_class=HTMLResponse) |
|
async def read_root(): |
|
|
|
html_content = """ |
|
<!DOCTYPE html> |
|
<html lang="en"> |
|
<head> |
|
<meta charset="UTF-8"> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
<title>Qwen2.5 Coder - AI Assistant</title> |
|
<script src="https://cdn.tailwindcss.com"></script> |
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"> |
|
<style> |
|
:root { |
|
--primary: #2563eb; |
|
--primary-dark: #1d4ed8; |
|
--dark: #1e293b; |
|
--light: #f8fafc; |
|
--gray: #94a3b8; |
|
--success: #10b981; |
|
} |
|
.chat-container { |
|
height: calc(100vh - 120px); |
|
} |
|
.message-user { |
|
background-color: white; |
|
border-left: 4px solid var(--primary); |
|
} |
|
.message-assistant { |
|
background-color: white; |
|
border-left: 4px solid var(--success); |
|
} |
|
.typing-indicator span { |
|
display: inline-block; |
|
width: 8px; |
|
height: 8px; |
|
background-color: var(--gray); |
|
border-radius: 50%; |
|
margin-right: 4px; |
|
animation: bounce 1.4s infinite ease-in-out; |
|
} |
|
.typing-indicator span:nth-child(2) { |
|
animation-delay: 0.2s; |
|
} |
|
.typing-indicator span:nth-child(3) { |
|
animation-delay: 0.4s; |
|
} |
|
@keyframes bounce { |
|
0%, 60%, 100% { transform: translateY(0); } |
|
30% { transform: translateY(-5px); } |
|
} |
|
.code-block { |
|
background-color: #0f172a; |
|
color: #e2e8f0; |
|
font-family: 'Fira Code', monospace; |
|
} |
|
.sidebar { |
|
transition: transform 0.3s ease; |
|
} |
|
@media (max-width: 768px) { |
|
.sidebar { |
|
transform: translateX(-100%); |
|
position: fixed; |
|
z-index: 50; |
|
height: 100vh; |
|
} |
|
.sidebar-open { |
|
transform: translateX(0); |
|
} |
|
} |
|
</style> |
|
</head> |
|
<body> |
|
<div class="flex h-screen overflow-hidden"> |
|
<!-- Sidebar --> |
|
<div class="sidebar bg-white w-64 border-r border-gray-200 flex flex-col md:relative absolute"> |
|
<div class="p-4 border-b border-gray-200 flex items-center"> |
|
<div class="w-10 h-10 rounded-full bg-blue-500 flex items-center justify-center text-white font-bold">Q</div> |
|
<div class="ml-3"> |
|
<h2 class="font-semibold">Qwen2.5 Coder</h2> |
|
<p class="text-xs text-gray-500">32B Instruct</p> |
|
</div> |
|
<button id="close-sidebar" class="ml-auto md:hidden text-gray-500"> |
|
<i class="fas fa-times"></i> |
|
</button> |
|
</div> |
|
<div class="p-4 border-b border-gray-200"> |
|
<button id="new-chat" class="w-full bg-blue-500 hover:bg-blue-600 text-white py-2 px-4 rounded-md flex items-center justify-center"> |
|
<i class="fas fa-plus mr-2"></i> New Chat |
|
</button> |
|
</div> |
|
<div class="flex-1 overflow-y-auto p-2"> |
|
<div class="p-2 text-sm font-medium text-gray-500">Recent Chats</div> |
|
<div id="chat-history" class="space-y-1"> |
|
<!-- Chat history will be populated here --> |
|
</div> |
|
</div> |
|
<div class="p-4 border-t border-gray-200"> |
|
<div class="flex items-center"> |
|
<div class="w-8 h-8 rounded-full bg-gray-300 flex items-center justify-center"> |
|
<i class="fas fa-user text-gray-600"></i> |
|
</div> |
|
<div class="ml-2"> |
|
<p class="text-sm font-medium">User Account</p> |
|
</div> |
|
</div> |
|
</div> |
|
</div> |
|
<!-- Main Content --> |
|
<div class="flex-1 flex flex-col overflow-hidden"> |
|
<!-- Header --> |
|
<header class="bg-white border-b border-gray-200 p-4 flex items-center"> |
|
<button id="menu-button" class="mr-4 text-gray-500 md:hidden"> |
|
<i class="fas fa-bars"></i> |
|
</button> |
|
<h1 class="text-xl font-semibold">Chat with Qwen2.5 Coder</h1> |
|
<div class="ml-auto flex space-x-2"> |
|
<button class="p-2 rounded-full hover:bg-gray-100"> |
|
<i class="fas fa-cog text-gray-500"></i> |
|
</button> |
|
<button class="p-2 rounded-full hover:bg-gray-100"> |
|
<i class="fas fa-question-circle text-gray-500"></i> |
|
</button> |
|
</div> |
|
</header> |
|
<!-- Chat Area --> |
|
<div id="chat-area" class="chat-container flex-1 overflow-y-auto p-4 space-y-4"> |
|
<div class="message-assistant p-4 rounded-lg shadow-sm"> |
|
<div class="flex items-start"> |
|
<div class="w-8 h-8 rounded-full bg-green-500 flex items-center justify-center text-white mr-3"> |
|
<i class="fas fa-robot"></i> |
|
</div> |
|
<div class="flex-1"> |
|
<p class="font-medium text-gray-700">Qwen2.5 Coder</p> |
|
<p class="mt-1 text-gray-800">Hello! I'm Qwen2.5 Coder, a 32B parameter AI assistant specialized in coding and technical questions. How can I help you today?</p> |
|
</div> |
|
</div> |
|
</div> |
|
</div> |
|
<!-- Input Area --> |
|
<div class="border-t border-gray-200 p-4 bg-white"> |
|
<div class="flex items-center mb-2"> |
|
<button class="p-2 rounded-full hover:bg-gray-100 text-gray-500 mr-1"> |
|
<i class="fas fa-paperclip"></i> |
|
</button> |
|
<button class="p-2 rounded-full hover:bg-gray-100 text-gray-500 mr-1"> |
|
<i class="fas fa-code"></i> |
|
</button> |
|
<button id="settings-button" class="p-2 rounded-full hover:bg-gray-100 text-gray-500"> |
|
<i class="fas fa-sliders-h"></i> |
|
</button> |
|
</div> |
|
<!-- Settings Panel --> |
|
<div id="settings-panel" class="hidden bg-gray-50 p-4 rounded-lg mb-4"> |
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-4"> |
|
<div> |
|
<label class="block text-sm font-medium text-gray-700 mb-1">System Message</label> |
|
<textarea id="system-message" class="w-full p-2 border border-gray-300 rounded-md text-sm" rows="2">You are a helpful AI assistant that specializes in coding and technical questions.</textarea> |
|
</div> |
|
<div> |
|
<label class="block text-sm font-medium text-gray-700 mb-1">Max Tokens</label> |
|
<input id="max-tokens" type="range" min="1" max="200000" value="512" class="w-full"> |
|
<div class="flex justify-between text-xs text-gray-500"> |
|
<span>1</span> |
|
<span>512</span> |
|
<span>200k</span> |
|
</div> |
|
</div> |
|
<div> |
|
<label class="block text-sm font-medium text-gray-700 mb-1">Temperature</label> |
|
<input id="temperature" type="range" min="0.1" max="4.0" step="0.1" value="0.7" class="w-full"> |
|
<div class="flex justify-between text-xs text-gray-500"> |
|
<span>0.1</span> |
|
<span>0.7</span> |
|
<span>4.0</span> |
|
</div> |
|
</div> |
|
<div> |
|
<label class="block text-sm font-medium text-gray-700 mb-1">Top-p</label> |
|
<input id="top-p" type="range" min="0.1" max="1.0" step="0.05" value="0.95" class="w-full"> |
|
<div class="flex justify-between text-xs text-gray-500"> |
|
<span>0.1</span> |
|
<span>0.95</span> |
|
<span>1.0</span> |
|
</div> |
|
</div> |
|
</div> |
|
</div> |
|
<div class="flex items-center"> |
|
<textarea id="message-input" class="flex-1 p-3 border border-gray-300 rounded-l-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" placeholder="Type your message here..." rows="1"></textarea> |
|
<button id="send-button" class="bg-blue-500 hover:bg-blue-600 text-white p-3 rounded-r-md"> |
|
<i class="fas fa-paper-plane"></i> |
|
</button> |
|
</div> |
|
<p class="text-xs text-gray-500 mt-2">Qwen2.5 Coder may produce inaccurate information about people, places, or facts.</p> |
|
</div> |
|
</div> |
|
</div> |
|
<script> |
|
// State management |
|
const state = { |
|
currentChatId: generateId(), |
|
chatHistory: {}, |
|
settings: { |
|
systemMessage: "You are a helpful AI assistant that specializes in coding and technical questions.", |
|
maxTokens: 512, |
|
temperature: 0.7, |
|
topP: 0.95 |
|
} |
|
}; |
|
// Initialize the app |
|
document.addEventListener('DOMContentLoaded', function() { |
|
// Load chat history from localStorage if available |
|
const savedHistory = localStorage.getItem('chatHistory'); |
|
if (savedHistory) { |
|
state.chatHistory = JSON.parse(savedHistory); |
|
renderChatHistory(); |
|
} |
|
// Create initial chat |
|
if (!state.chatHistory[state.currentChatId]) { |
|
state.chatHistory[state.currentChatId] = { |
|
id: state.currentChatId, |
|
title: "New Chat", |
|
messages: [ |
|
{ |
|
role: "assistant", |
|
content: "Hello! I'm Qwen2.5 Coder, a 32B parameter AI assistant specialized in coding and technical questions. How can I help you today?" |
|
} |
|
], |
|
createdAt: new Date().toISOString() |
|
}; |
|
saveChatHistory(); |
|
} |
|
// UI event listeners |
|
setupEventListeners(); |
|
}); |
|
// Helper functions |
|
function generateId() { |
|
return Date.now().toString(36) + Math.random().toString(36).substring(2); |
|
} |
|
function saveChatHistory() { |
|
localStorage.setItem('chatHistory', JSON.stringify(state.chatHistory)); |
|
} |
|
function renderChatHistory() { |
|
const chatHistoryContainer = document.getElementById('chat-history'); |
|
chatHistoryContainer.innerHTML = ''; |
|
// Sort chats by creation date (newest first) |
|
const sortedChats = Object.values(state.chatHistory).sort((a, b) => |
|
new Date(b.createdAt) - new Date(a.createdAt) |
|
); |
|
sortedChats.forEach(chat => { |
|
const chatElement = document.createElement('div'); |
|
chatElement.className = `p-2 hover:bg-gray-100 rounded-md cursor-pointer flex items-center ${chat.id === state.currentChatId ? 'bg-gray-100' : ''}`; |
|
chatElement.innerHTML = ` |
|
<i class="fas fa-comment mr-2 text-gray-500"></i> |
|
<span class="truncate">${chat.title}</span> |
|
`; |
|
chatElement.addEventListener('click', () => loadChat(chat.id)); |
|
chatHistoryContainer.appendChild(chatElement); |
|
}); |
|
} |
|
function loadChat(chatId) { |
|
state.currentChatId = chatId; |
|
renderChatHistory(); |
|
renderChatMessages(); |
|
} |
|
function renderChatMessages() { |
|
const chatArea = document.getElementById('chat-area'); |
|
chatArea.innerHTML = ''; |
|
const currentChat = state.chatHistory[state.currentChatId]; |
|
if (!currentChat) return; |
|
currentChat.messages.forEach(message => { |
|
const messageDiv = document.createElement('div'); |
|
messageDiv.className = message.role === 'user' ? |
|
'message-user p-4 rounded-lg shadow-sm' : |
|
'message-assistant p-4 rounded-lg shadow-sm'; |
|
messageDiv.innerHTML = ` |
|
<div class="flex items-start"> |
|
<div class="w-8 h-8 rounded-full ${message.role === 'user' ? 'bg-blue-500' : 'bg-green-500'} flex items-center justify-center text-white mr-3"> |
|
<i class="fas ${message.role === 'user' ? 'fa-user' : 'fa-robot'}"></i> |
|
</div> |
|
<div class="flex-1"> |
|
<p class="font-medium text-gray-700">${message.role === 'user' ? 'You' : 'Qwen2.5 Coder'}</p> |
|
<div class="mt-1 text-gray-800">${formatMessageContent(message.content)}</div> |
|
</div> |
|
</div> |
|
`; |
|
chatArea.appendChild(messageDiv); |
|
}); |
|
chatArea.scrollTop = chatArea.scrollHeight; |
|
} |
|
function formatMessageContent(content) { |
|
// Simple formatting for code blocks (replace with proper markdown parsing if needed) |
|
return content.replace(/```([\s\S]*?)```/g, '<div class="code-block mt-2 p-3 rounded-md text-sm overflow-x-auto"><pre><code>$1</code></pre></div>'); |
|
} |
|
function setupEventListeners() { |
|
// Toggle sidebar on mobile |
|
document.getElementById('menu-button').addEventListener('click', function() { |
|
document.querySelector('.sidebar').classList.add('sidebar-open'); |
|
}); |
|
document.getElementById('close-sidebar').addEventListener('click', function() { |
|
document.querySelector('.sidebar').classList.remove('sidebar-open'); |
|
}); |
|
// Toggle settings panel |
|
document.getElementById('settings-button').addEventListener('click', function() { |
|
document.getElementById('settings-panel').classList.toggle('hidden'); |
|
}); |
|
// Auto-resize textarea |
|
const textarea = document.getElementById('message-input'); |
|
textarea.addEventListener('input', function() { |
|
this.style.height = 'auto'; |
|
this.style.height = (this.scrollHeight) + 'px'; |
|
}); |
|
// New chat button |
|
document.getElementById('new-chat').addEventListener('click', function() { |
|
const newChatId = generateId(); |
|
state.currentChatId = newChatId; |
|
state.chatHistory[newChatId] = { |
|
id: newChatId, |
|
title: "New Chat", |
|
messages: [ |
|
{ |
|
role: "assistant", |
|
content: "Hello! I'm Qwen2.5 Coder, a 32B parameter AI assistant specialized in coding and technical questions. How can I help you today?" |
|
} |
|
], |
|
createdAt: new Date().toISOString() |
|
}; |
|
saveChatHistory(); |
|
renderChatHistory(); |
|
renderChatMessages(); |
|
document.querySelector('.sidebar').classList.remove('sidebar-open'); |
|
}); |
|
// Send message |
|
document.getElementById('send-button').addEventListener('click', sendMessage); |
|
// Allow pressing Enter to send message (Shift+Enter for new line) |
|
textarea.addEventListener('keydown', function(e) { |
|
if (e.key === 'Enter' && !e.shiftKey) { |
|
e.preventDefault(); |
|
sendMessage(); |
|
} |
|
}); |
|
// Settings change listeners |
|
document.getElementById('system-message').addEventListener('change', function() { |
|
state.settings.systemMessage = this.value; |
|
}); |
|
document.getElementById('max-tokens').addEventListener('input', function() { |
|
state.settings.maxTokens = parseInt(this.value); |
|
// Update the displayed value |
|
this.parentNode.querySelector('span:nth-child(2)').textContent = this.value; |
|
}); |
|
document.getElementById('temperature').addEventListener('input', function() { |
|
state.settings.temperature = parseFloat(this.value); |
|
this.parentNode.querySelector('span:nth-child(2)').textContent = this.value; |
|
}); |
|
document.getElementById('top-p').addEventListener('input', function() { |
|
state.settings.topP = parseFloat(this.value); |
|
this.parentNode.querySelector('span:nth-child(2)').textContent = this.value; |
|
}); |
|
} |
|
async function sendMessage() { |
|
const textarea = document.getElementById('message-input'); |
|
const message = textarea.value.trim(); |
|
if (message) { |
|
// Clear input and adjust height |
|
textarea.value = ''; |
|
textarea.style.height = 'auto'; |
|
// Add user message to chat |
|
addMessageToChat('user', message); |
|
// Show typing indicator |
|
showTypingIndicator(); |
|
try { |
|
// Prepare the request data |
|
const requestData = { |
|
message: message, |
|
history: getMessageHistory(), |
|
system_message: state.settings.systemMessage, |
|
max_tokens: state.settings.maxTokens, |
|
temperature: state.settings.temperature, |
|
top_p: state.settings.topP |
|
}; |
|
// Call the API |
|
const response = await fetch('/api/chat', { |
|
method: 'POST', |
|
headers: { |
|
'Content-Type': 'application/json', |
|
}, |
|
body: JSON.stringify(requestData) |
|
}); |
|
// Handle streaming response |
|
const reader = response.body.getReader(); |
|
const decoder = new TextDecoder(); |
|
let assistantResponse = ''; |
|
let responseId = generateId(); |
|
while(true) { |
|
const { done, value } = await reader.read(); |
|
if(done) break; |
|
const chunk = decoder.decode(value); |
|
assistantResponse += chunk; |
|
// Update the assistant message in real-time |
|
updateAssistantMessage(responseId, assistantResponse); |
|
} |
|
// Finalize the message |
|
finalizeMessage(responseId, assistantResponse); |
|
updateChatTitle(message); |
|
} catch (error) { |
|
console.error('Error:', error); |
|
addMessageToChat('assistant', "Sorry, I encountered an error. Please try again."); |
|
} finally { |
|
hideTypingIndicator(); |
|
} |
|
} |
|
} |
|
function addMessageToChat(role, content, id) { |
|
const messageId = id || generateId(); |
|
const currentChat = state.chatHistory[state.currentChatId]; |
|
currentChat.messages.push({ |
|
id: messageId, |
|
role: role, |
|
content: content |
|
}); |
|
saveChatHistory(); |
|
renderChatMessages(); |
|
return messageId; |
|
} |
|
function updateAssistantMessage(id, content) { |
|
const messageElement = document.querySelector(`[data-message-id="${id}"]`); |
|
if(messageElement) { |
|
messageElement.querySelector('.message-content').innerHTML = formatMessageContent(content); |
|
messageElement.scrollIntoView({ behavior: 'smooth' }); |
|
} |
|
} |
|
function finalizeMessage(id, content) { |
|
const currentChat = state.chatHistory[state.currentChatId]; |
|
const message = currentChat.messages.find(m => m.id === id); |
|
if(message) { |
|
message.content = content; |
|
saveChatHistory(); |
|
} |
|
} |
|
function showTypingIndicator() { |
|
const typingIndicator = document.createElement('div'); |
|
typingIndicator.className = 'message-assistant p-4 rounded-lg shadow-sm'; |
|
typingIndicator.innerHTML = ` |
|
<div class="flex items-start"> |
|
<div class="w-8 h-8 rounded-full bg-green-500 flex items-center justify-center text-white mr-3"> |
|
<i class="fas fa-robot"></i> |
|
</div> |
|
<div class="flex-1"> |
|
<p class="font-medium text-gray-700">Qwen2.5 Coder</p> |
|
<div class="typing-indicator mt-1"> |
|
<span></span> |
|
<span></span> |
|
<span></span> |
|
</div> |
|
</div> |
|
</div> |
|
`; |
|
document.getElementById('chat-area').appendChild(typingIndicator); |
|
} |
|
function hideTypingIndicator() { |
|
const typingIndicators = document.getElementsByClassName('typing-indicator'); |
|
while(typingIndicators.length > 0) { |
|
typingIndicators[0].parentNode.parentNode.parentNode.remove(); |
|
} |
|
} |
|
function getMessageHistory() { |
|
const currentChat = state.chatHistory[state.currentChatId]; |
|
return currentChat.messages |
|
.filter(msg => msg.role !== 'system') |
|
.map(msg => [msg.role === 'user' ? msg.content : '', msg.role === 'assistant' ? msg.content : '']); |
|
} |
|
</script> |
|
</body> |
|
</html> |
|
""" |
|
return HTMLResponse(content=html_content) |
|
|
|
if __name__ == "__main__": |
|
uvicorn.run(app, host="0.0.0.0", port=7860) |
