feat: first import

Dockerfile

@@ -0,0 +1,32 @@
+# Grab a fresh copy of the Python image
+FROM python:3.10-slim
+
+# Install build and runtime dependencies
+RUN apt-get update && \
+ apt-get install -y \
+ libopenblas-dev \
+ ninja-build \
+ build-essential \
+ curl
+
+RUN pip install -U pip setuptools wheel && \
+ CMAKE_ARGS="-DLLAMA_OPENBLAS=on" FORCE_CMAKE=1 pip install llama-cpp-python[server]
+
+# Download model
+RUN mkdir model && \
+ curl -L https://huggingface.co/TheBloke/orca_mini_v2_7B-GGML/resolve/main/orca-mini-v2_7b.ggmlv3.q4_0.bin -o model/ggml-model-q4_0.bin
+
+COPY ./start_server.sh ./start_server.sh
+
+# Make the server start script executable
+RUN chmod +x ./start_server.sh
+
+# Set environment variable for the host
+ENV HOST=0.0.0.0
+ENV PORT=7860
+
+# Expose a port for the server
+EXPOSE ${PORT}
+
+# Run the server start script
+CMD ["/bin/sh", "./start_server.sh"] 

start_server.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# For mlock support
+ulimit -l unlimited
+
+python3 -B -m llama_cpp.server --model model/ggml-model-q4_0.bin