图像生成-vllm部署gpt oss 20B模型-29

docker run -it --rm --gpus all \
--name gpt_oss_20b_xxxx \
-v /data3/xxxx:/data3/xxxx \
-p 18010:8000 \
--ipc=host \
--entrypoint bash \
vllm/vllm-openai

.bashrc配置代理

pip install --pre vllm==0.10.1+gptoss \
  --extra-index-url https://wheels.vllm.ai/gpt-oss/ \
  --extra-index-url https://download.pytorch.org/whl/nightly/cu128

export CUDA_VISIBLE_DEVICES=7  # 80Gb

python3 -m vllm.entrypoints.openai.api_server \
  --model /data3/xxxx/gpt-oss-20b \
  --served-model-name gpt_oss_20b\
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 1 \
  --gpu-memory-utilization 0.95 \
  --max-model-len 4096 \
  --enable-auto-tool-choice \
  --tool-call-parser hermes \
  --trust-remote-code

微调
llamafactory升级到最新版本 其他参数以及指令 参考官方实现即可

git clone --depth 1 GitHub - hiyouga/LLaMA-Factory: Unified Efficient Fine-Tuning of 100+ LLMs & VLMs (ACL 2024)cd LLaMA-Factory
pip install -e ".[torch,metrics]" --no-build-isolation
pip install "transformers==4.55.0"