国产部署大模型

# physics :  Atlas300i A2 64G   * 2




export IMAGE=quay.io/ascend/vllm-ascend:v0.23.0
export NAME=vllm-ascend
docker run --rm \
    --name $NAME \
    --shm-size=16g \
    --net=host \
    --device /dev/davinci0 \
    --device /dev/davinci1 \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
    -v /usr/local/dcmi:/usr/local/dcmi \
    -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /root/.cache:/root/.cache \
    -it $IMAGE bash



# After entering the container, verify that vLLM and vLLM-Ascend can be imported:
python -c "import vllm, vllm_ascend; print('vllm and vllm_ascend are ready')"





#!/bin/sh
# Load model from ModelScope to speed up download
# https://www.modelscope.cn/models/Eco-Tech/Qwen3.8-27B-w8a8
export VLLM_USE_MODELSCOPE=True
# Size of the shared buffer (in MB) used by HCCL for NPU-to-NPU collective communication
export HCCL_BUFFSIZE=512
# To reduce memory fragmentation and avoid out of memory
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
# Model weight path; can be a ModelScope model id (e.g., Eco‑Tech/Qwen3.8‑27B‑w8a8) or a local directory path
export MODEL_PATH=Eco-Tech/Qwen3.8-27B-w8a8

vllm serve $MODEL_PATH \
    --host 0.0.0.0 \
    --port 8000 \
    --data-parallel-size 1 \
    --tensor-parallel-size 2 \
    --quantization ascend-w8a8 \
    --served-model-name qwen3.8 \
    --max-num-seqs 8 \
    --max-model-len 131072 \
    --max-num-batched-tokens 16384 \
    --block-size 128 \
    --enable-chunked-prefill \
    --trust-remote-code \
    --enable-prefix-caching \
    --gpu-memory-utilization 0.78 \
    --additional-config '{"enable_cpu_binding":true}'






# test
curl http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "qwen3.8",
        "messages": [
            {"role": "user", "content": "The future of AI is"}
        ],
        "max_completion_tokens": 1024,
        "temperature": 1.0,
        "top_p": 0.95
    }'