Spaces:
Paused
Paused
File size: 989 Bytes
69c6372 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
#!/bin/bash
# Check if MODEL_NAME is set and has a valid value
if [[ -z "$MODEL_NAME" ]]; then
echo "Error: MODEL_NAME is not set."
exit 1
fi
if [[ "$MODEL_NAME" != "meta-llama/Llama-3.2-3B-Instruct" && "$MODEL_NAME" != "sail/Sailor-4B-Chat" ]]; then
echo "Error: Invalid value for MODEL_NAME. Valid values are:"
echo " - meta-llama/Llama-3.2-3B-Instruct"
echo " - sail/Sailor-4B-Chat"
exit 1
fi
# Check if MODEL_REV is set
if [[ -z "$MODEL_REV" ]]; then
echo "Error: MODEL_REV is not set."
exit 1
fi
printf "Running %s using vLLM OpenAI compatible API Server at port %s\n" $MODEL_NAME "7860"
# Run the Python script with the validated environment variables
python -u /app/openai_compatible_api_server.py \
--model "${MODEL_NAME}" \
--revision "${MODEL_REV}" \
--seed 42 \
--host 0.0.0.0 \
--port 7860 \
--max-num-batched-tokens 32768 \
--max-model-len 32768 \
--dtype float16 \
--enforce-eager \
--gpu-memory-utilization 0.9
|