wip

Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3.10
+
+RUN apt-get update && \
+ apt-get install -y \
+ build-essential \
+ libssl-dev \
+ libffi-dev \
+ libbz2-dev \
+ liblzma-dev \
+ zlib1g-dev \
+ libsqlite3-dev \
+ wget \
+ curl \
+ git \
+ cmake \
+ libblis64-4 \
+ libblis64-4-openmp \
+ libblis64-4-pthread \
+ libblis64-4-serial \
+ libblis64-dev \
+ libblis64-openmp-dev \
+ libblis64-pthread-dev \
+ libblis64-serial-dev && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+ENV GOMP_CPU_AFFINITY="0-19"
+ENV BLIS_NUM_THREADS=14
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir --upgrade pip
+RUN CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=FLAME" \
+ pip install llama-cpp-python
+
+COPY --chown=user ./app.py /app/app.py
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,24 @@
+import gradio as gr
+from llama_cpp import Llama
+
+print("Downloading model")
+llm = Llama.from_pretrained(
+ repo_id="bartowski/gemma-2-2b-it-abliterated-GGUF",
+ filename="gemma-2-2b-it-abliterated-IQ4_XS.gguf",
+ numa=True,
+ n_ctx=4096,
+)
+
+
+def respond(prompt: str):
+ stream = llm.create_chat_completion(stream=True, messages=[{"role": "user", "content": prompt}])
+
+ response = ""
+ for chunk in stream:
+ if "content" in chunk["choices"][0]["delta"]:
+ response += chunk["choices"][0]["delta"]["content"]
+ yield response
+
+
+demo = gr.Interface(fn=respond, inputs=[gr.TextArea("What is the capital of France?")], outputs=[gr.TextArea()])
+demo.launch(server_name="0.0.0.0", server_port=7860)