Revert to use CPU resources in the quantization process

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: FastAPI",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "main:app",
+                "--reload",
+                "--port",
+                "5000",
+                "--host",
+                "0.0.0.0"
+            ],
+            "jinja": true,
+            "cwd": "${workspaceFolder}",
+        }
+    ]
+}

Dockerfile

@@ -17,8 +17,8 @@ RUN pip install --upgrade pip
 
 RUN pip install -U setuptools wheel
 
-# Install torch
-RUN pip install -U torch torchvision torchaudio
+# Install torch cpu version
+RUN pip install -U torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 
 # Copy the requirements file into the container
 COPY --chown=user requirements.txt .

main.py

@@ -6,7 +6,7 @@ from tempfile import NamedTemporaryFile
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import RedirectResponse, FileResponse
-from .dto import AWQConvertionRequest, GGUFConvertionRequest, GPTQConvertionRequest
+from dto import AWQConvertionRequest, GGUFConvertionRequest, GPTQConvertionRequest
 
 ### FastAPI Initialization
 @asynccontextmanager
@@ -26,7 +26,7 @@ def redirect_to_docs():
 def convert(request: AWQConvertionRequest)->Union[FileResponse, dict]:
     
     try:
-        model = AutoAWQForCausalLM.from_pretrained(request.hf_model_name)
+        model = AutoAWQForCausalLM.from_pretrained(request.hf_model_name, device_map="cpu", trust_remote_code=True)
     except TypeError as e:
         raise HTTPException(status_code=400, detail=f"Is this model supported by AWQ Quantization? Check:https://github.com/mit-han-lab/llm-awq?tab=readme-ov-file {e}")
     

requirements.txt

@@ -8,7 +8,7 @@ pydantic
 fastapi[standard]
 transformers
 huggingface_hub[hf_xet]
-autoawq>=0.2.8
+autoawq[cpu]>=0.2.8
 pytest
 requests
 environs