|
--- |
|
license: apache-2.0 |
|
pipeline_tag: text-generation |
|
library_name: transformers |
|
--- |
|
|
|
# AMD-Llama-135m-code-GGUF |
|
|
|
|
|
## Introduction |
|
AMD-Llama-135m is a language model trained on AMD MI250 GPUs. Based on LLaMA2 model architecture, this model can be smoothly loaded as LlamaForCausalLM with huggingface transformers. Furthermore, we use the same tokenizer as LLaMA2, enabling it to be a draft model of speculative decoding for LLaMA2 and CodeLlama. |
|
|
|
|
|
## Quickstart |
|
AMD-Llama-135m-code-GGUF can be loaded and used via Llama.cpp, here is a program with GUI. |
|
|
|
```bash |
|
pip install PyQt5 llama-cpp-python pymupdf |
|
``` |
|
|
|
```python |
|
import sys |
|
import os |
|
from PyQt5.QtWidgets import (QApplication, QWidget, QLabel, QPushButton, |
|
QLineEdit, QTextEdit, QVBoxLayout, QHBoxLayout, |
|
QFileDialog, QProgressBar, QMessageBox, QMenu) |
|
from PyQt5.QtCore import Qt, QThread, pyqtSignal |
|
from llama_cpp import Llama |
|
import fitz # For PDF processing |
|
|
|
class Worker(QThread): |
|
finished = pyqtSignal(str) |
|
progress = pyqtSignal(int, int) |
|
|
|
def __init__(self, model, messages, max_tokens): |
|
super().__init__() |
|
self.model = model |
|
self.messages = messages |
|
self.max_tokens = max_tokens |
|
|
|
def run(self): |
|
try: |
|
response = self.model.create_chat_completion( |
|
messages=self.messages, |
|
max_tokens=self.max_tokens, |
|
temperature=0.7, |
|
stream=True |
|
) |
|
|
|
total_tokens = 0 |
|
full_response = "" |
|
for chunk in response: |
|
if "choices" in chunk: |
|
content = chunk["choices"][0]["delta"].get("content", "") |
|
full_response += content |
|
total_tokens += 1 |
|
self.progress.emit(total_tokens, self.max_tokens) |
|
self.finished.emit(full_response) |
|
except Exception as e: |
|
self.finished.emit(f"Error generating response: {str(e)}") |
|
|
|
class ChatbotGUI(QWidget): |
|
def __init__(self): |
|
super().__init__() |
|
self.setWindowTitle("Chatbot GUI") |
|
self.resize(800, 600) |
|
|
|
self.model = None |
|
self.messages = [ |
|
{"role": "system", "content": "You are a helpful AI assistant."} |
|
] |
|
self.thread_count = 12 |
|
self.pdf_content = "" |
|
|
|
self.initUI() |
|
|
|
def initUI(self): |
|
# Model loading section |
|
model_label = QLabel("Model: No model loaded") |
|
load_button = QPushButton("Load GGUF Model") |
|
load_button.clicked.connect(self.load_model) |
|
|
|
model_layout = QHBoxLayout() |
|
model_layout.addWidget(model_label) |
|
model_layout.addWidget(load_button) |
|
|
|
# PDF upload section |
|
pdf_label = QLabel("PDF: No PDF loaded") |
|
upload_pdf_button = QPushButton("Upload PDF") |
|
upload_pdf_button.clicked.connect(self.upload_pdf) |
|
|
|
pdf_layout = QHBoxLayout() |
|
pdf_layout.addWidget(pdf_label) |
|
pdf_layout.addWidget(upload_pdf_button) |
|
|
|
# Thread count section |
|
thread_label = QLabel(f"Thread Count: {self.thread_count}") |
|
self.thread_input = QLineEdit() |
|
self.thread_input.setPlaceholderText("Enter new thread count") |
|
update_thread_button = QPushButton("Update Threads") |
|
update_thread_button.clicked.connect(self.update_thread_count) |
|
|
|
thread_layout = QHBoxLayout() |
|
thread_layout.addWidget(thread_label) |
|
thread_layout.addWidget(self.thread_input) |
|
thread_layout.addWidget(update_thread_button) |
|
|
|
# Chat display |
|
self.chat_display = QTextEdit() |
|
self.chat_display.setReadOnly(True) |
|
self.chat_display.setContextMenuPolicy(Qt.CustomContextMenu) |
|
self.chat_display.customContextMenuRequested.connect(self.show_context_menu) |
|
|
|
# User input |
|
self.user_input = QLineEdit() |
|
self.user_input.returnPressed.connect(self.send_message) |
|
send_button = QPushButton("Send") |
|
send_button.clicked.connect(self.send_message) |
|
|
|
input_layout = QHBoxLayout() |
|
input_layout.addWidget(self.user_input) |
|
input_layout.addWidget(send_button) |
|
|
|
# Progress bar |
|
self.progress_bar = QProgressBar() |
|
self.progress_bar.hide() |
|
|
|
# Clear conversation button |
|
clear_button = QPushButton("Clear Conversation") |
|
clear_button.clicked.connect(self.clear_conversation) |
|
|
|
# Main layout |
|
main_layout = QVBoxLayout() |
|
main_layout.addLayout(model_layout) |
|
main_layout.addLayout(pdf_layout) # PDF before threads |
|
main_layout.addLayout(thread_layout) |
|
main_layout.addWidget(self.chat_display) |
|
main_layout.addWidget(self.progress_bar) |
|
main_layout.addLayout(input_layout) |
|
main_layout.addWidget(clear_button) |
|
|
|
self.setLayout(main_layout) |
|
|
|
def load_model(self): |
|
model_path, _ = QFileDialog.getOpenFileName(self, "Load GGUF Model", "", "GGUF Files (*.gguf)") |
|
if model_path: |
|
try: |
|
self.model = Llama(model_path=model_path, n_ctx=2048, n_gpu_layers=-1, n_threads=self.thread_count) |
|
model_name = os.path.basename(model_path) |
|
self.layout().itemAt(0).itemAt(0).widget().setText(f"Model: {model_name}") |
|
QMessageBox.information(self, "Success", "Model loaded successfully!") |
|
except Exception as e: |
|
error_message = f"Error loading model: {str(e)}" |
|
QMessageBox.critical(self, "Error", error_message) |
|
|
|
def update_thread_count(self): |
|
try: |
|
new_thread_count = int(self.thread_input.text()) |
|
if new_thread_count > 0: |
|
self.thread_count = new_thread_count |
|
self.layout().itemAt(2).itemAt(0).widget().setText(f"Thread Count: {self.thread_count}") # Updated index |
|
self.thread_input.clear() |
|
if self.model: |
|
self.model.set_thread_count(self.thread_count) |
|
QMessageBox.information(self, "Success", f"Thread count updated to {self.thread_count}") |
|
else: |
|
raise ValueError("Thread count must be a positive integer") |
|
except ValueError as e: |
|
QMessageBox.warning(self, "Invalid Input", str(e)) |
|
|
|
def upload_pdf(self): |
|
pdf_path, _ = QFileDialog.getOpenFileName(self, "Upload PDF", "", "PDF Files (*.pdf)") |
|
if pdf_path: |
|
try: |
|
doc = fitz.open(pdf_path) |
|
self.pdf_content = "" |
|
for page in doc: |
|
self.pdf_content += page.get_text() |
|
self.layout().itemAt(1).itemAt(0).widget().setText(f"PDF: {os.path.basename(pdf_path)}") # Updated index |
|
QMessageBox.information(self, "Success", "PDF loaded successfully!") |
|
except Exception as e: |
|
QMessageBox.critical(self, "Error", f"Error loading PDF: {str(e)}") |
|
|
|
def send_message(self): |
|
user_message = self.user_input.text() |
|
if user_message and self.model: |
|
self.messages.append({"role": "user", "content": user_message}) |
|
self.update_chat_display(f"You: {user_message}") |
|
self.user_input.clear() |
|
|
|
max_tokens = 1000 |
|
self.progress_bar.show() |
|
self.progress_bar.setRange(0, max_tokens) |
|
self.progress_bar.setValue(0) |
|
|
|
# Add PDF content if available |
|
if self.pdf_content: |
|
self.messages.append({"role": "user", "content": self.pdf_content}) |
|
|
|
self.worker = Worker(self.model, self.messages, max_tokens) |
|
self.worker.finished.connect(self.on_response_finished) |
|
self.worker.progress.connect(self.on_response_progress) |
|
self.worker.start() |
|
|
|
def on_response_finished(self, assistant_message): |
|
self.progress_bar.hide() |
|
self.messages.append({"role": "assistant", "content": assistant_message}) |
|
self.update_chat_display(f"Assistant: {assistant_message}") |
|
|
|
# Python Code Download |
|
if assistant_message.startswith("```python") and assistant_message.endswith("```"): |
|
self.offer_code_download(assistant_message) |
|
|
|
def on_response_progress(self, current_tokens, total_tokens): |
|
self.progress_bar.setValue(current_tokens) |
|
|
|
def offer_code_download(self, code): |
|
reply = QMessageBox.question(self, "Download Code", |
|
"The assistant generated Python code. Do you want to download it?", |
|
QMessageBox.Yes | QMessageBox.No) |
|
if reply == QMessageBox.Yes: |
|
file_path, _ = QFileDialog.getSaveFileName(self, "Save Python Code", "code.py", "Python Files (*.py)") |
|
if file_path: |
|
try: |
|
with open(file_path, "w") as f: |
|
f.write(code.strip("```python").strip("```")) |
|
QMessageBox.information(self, "Success", "Code saved successfully!") |
|
except Exception as e: |
|
QMessageBox.critical(self, "Error", f"Error saving code: {str(e)}") |
|
|
|
def update_chat_display(self, message): |
|
self.chat_display.append(message + "\n") |
|
self.chat_display.verticalScrollBar().setValue(self.chat_display.verticalScrollBar().maximum()) |
|
|
|
def clear_conversation(self): |
|
self.messages = [ |
|
{"role": "system", "content": "You are a helpful AI assistant."} |
|
] |
|
self.chat_display.clear() |
|
self.pdf_content = "" # Clear PDF content |
|
self.layout().itemAt(1).itemAt(0).widget().setText("PDF: No PDF loaded") # Updated index |
|
|
|
def show_context_menu(self, point): |
|
menu = QMenu(self) |
|
copy_action = menu.addAction("Copy") |
|
copy_action.triggered.connect(self.copy_text) |
|
menu.exec_(self.chat_display.mapToGlobal(point)) |
|
|
|
def copy_text(self): |
|
cursor = self.chat_display.textCursor() |
|
if cursor.hasSelection(): |
|
text = cursor.selectedText() |
|
QApplication.clipboard().setText(text) |
|
|
|
|
|
if __name__ == "__main__": |
|
app = QApplication(sys.argv) |
|
gui = ChatbotGUI() |
|
gui.show() |
|
sys.exit(app.exec_()) |
|
``` |
|
|
|
## Training and finetuning cost |
|
It takes 6 days to pretrain AMD-Llama-135m on 4 MI250 nodes each of which has 4 MI250 GPUs (8 virtual GPU cards, 64G memory for each). |
|
It takes 4 days to finetune AMD-Llama-135m-code on 4 MI250 GPUs. |
|
It takes 11T disk space to store raw and processed SlimPajama, project gutenberg and Starcoder datasets. |
|
|
|
#### License |
|
Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All Rights Reserved. |
|
|
|
Licensed under the Apache License, Version 2.0 (the "License"); |
|
you may not use this file except in compliance with the License. |
|
You may obtain a copy of the License at |
|
|
|
http://www.apache.org/licenses/LICENSE-2.0 |
|
|
|
Unless required by applicable law or agreed to in writing, software |
|
distributed under the License is distributed on an "AS IS" BASIS, |
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
See the License for the specific language governing permissions and |
|
limitations under the License. |
