First implementation

.gitignore

@@ -0,0 +1,3 @@
+.vscode
+.venv
+__pycache__

Dockerfile

@@ -0,0 +1,15 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY ./vigostral-7b-chat.Q6_K.gguf /code/vigostral-7b-chat.Q6_K.gguf
+COPY ./main.py /code/main.py
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,88 @@
+import uvicorn
+import json
+
+from ctransformers import AutoModelForCausalLM
+from fastapi import FastAPI, Form
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+from typing import List, Dict
+from fastapi.middleware.cors import CORSMiddleware
+
+#Model loading
+model = AutoModelForCausalLM.from_pretrained("vigostral-7b-chat.Q6_K.gguf",
+    model_type='llama',
+    threads = 3,
+)
+
+#Fast API
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins = ["*"],
+    allow_credentials = True,
+    allow_methods = ["*"],
+    allow_headers = ["*"],
+)
+
+def apply_chat_template(conversation: List):
+    formatted_conversation = ""
+
+    for turn in conversation:
+        role = turn.role.upper()
+        content = turn.content
+        if role == "SYSTEM":
+            formatted_conversation += "<s>[INST] <<SYS>>\n" + content + "\n<</SYS>>"
+        elif role == "ASSISTANT":
+            formatted_conversation += "\n[/INST] " + content + " </s>"
+        else:
+            formatted_conversation += "[INST] " + content + " [/INST]"
+
+    return formatted_conversation
+
+#Pydantic object
+class Message(BaseModel):
+    role: str
+    content: str
+
+class Validation(BaseModel):
+    messages: List[Message]
+    model: str
+    temperature: float
+    presence_penalty: float
+    top_p: float
+    frequency_penalty: float
+    stream: bool
+
+@app.post("/chat")
+async def stream(item: Validation):
+
+    prompt = apply_chat_template(item.messages)
+
+    def stream_json():
+        for text in model(
+            prompt,
+            temperature=item.temperature,
+            top_p=item.top_p,
+            presence_penalty=item.presence_penalty,
+            frequency_penalty=item.frequency_penalty,
+            stream=True
+            ):
+            yield json.dumps({
+                "object":"chat.completion.chunk",
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta" : {
+                            "content": text
+                        }
+                    }
+                ]
+            })
+
+    return StreamingResponse(stream_json(), media_type="application/json")
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+
+

requirements.txt

@@ -0,0 +1,4 @@
+fastapi
+pydantic
+uvicorn
+ctransformers

test.ipynb

@@ -0,0 +1,67 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/ilhooq/Dev/Python/IA/vigostral-chat/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from ctransformers import AutoModelForCausalLM\n",
+    "\n",
+    "#Model loading\n",
+    "llm = AutoModelForCausalLM.from_pretrained(\"/home/ilhooq/Stockage/IA-models/vigostral-7b-chat.Q6_K.gguf\",\n",
+    "    model_type='mistral',\n",
+    "    max_new_tokens = 1096,\n",
+    "    threads = 3,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "llama\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(llm.model_type)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

vigostral-7b-chat.Q6_K.gguf

@@ -0,0 +1 @@
+/home/ilhooq/Stockage/IA-models/vigostral-7b-chat.Q6_K.gguf