大模型部署

export IMAGE=quay.io/ascend/vllm-ascend:v0.23.0
export NAME=vllm-ascend
docker run --rm \
    --name $NAME \
    --shm-size=16g \
    --net=host \
    --device /dev/davinci2 \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
    -v /usr/local/dcmi:/usr/local/dcmi \
    -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /root/.cache:/root/.cache \
    -it $IMAGE bash





# After entering the container, verify that vLLM and vLLM-Ascend can be imported:
python -c "import vllm, vllm_ascend; print('vllm and vllm_ascend are ready')"




curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "qwen3.8",
    "messages": [
      {"role": "user", "content": "查询今天天津的气温"}
    ],
    "tools": [
      {
        "type": "function",
        "function": {
          "name": "get_weather",
          "description": "获取指定城市当前天气",
          "parameters": {
            "type": "object",
            "properties": {
              "city": {
                "type": "string",
                "description": "城市名称"
              }
            },
            "required": ["city"]
          }
        }
      }
    ],
    "tool_choice": "auto",
    "max_completion_tokens": 1024,
    "temperature": 1.0,
    "top_p": 0.95,
    "stream": true
  }'




export MODEL_PATH=/root/.cache/modelscope/models/Eco-Tech--Qwen3.8-27B-w8a8/snapshots/master
# https://recipes.vllm.ai/Qwen/Qwen3.8-27B?features=encoder_parallel%2Creasoning%2Clong_context

vllm serve $MODEL_PATH     --host 0.0.0.0     --port 8000     --tensor-parallel-size 2     --served-model-name qwen3.8     --max-num-seqs 10     --max-model-len 65536      --trust-remote-code     --gpu-memory-utilization 0.90     --mamba-ssm-cache-dtype bfloat16     --dtype bfloat16   --enable-auto-tool-choice   --tool-call-parser qwen3_xml --reasoning-parser qwen3












# 单卡

vllm serve $MODEL_PATH \
    --host 0.0.0.0 \
    --port 8000 \
    --served-model-name qwen3.8 \
    --max-num-seqs 10 \
    --max-model-len 65536 \
    --trust-remote-code \
    --gpu-memory-utilization 0.90 \
    --mamba-ssm-cache-dtype bfloat16 \
    --dtype bfloat16 \
    --enable-auto-tool-choice \
    --tool-call-parser qwen3_xml \
    --reasoning-parser qwen3