model-运行

python -m vllm.entrypoints.openai.api_server
--model /models/Qwen3-235B-A22B-Instruct-2507
--host 0.0.0.0 --port 9995
--served-model-name deepseek-r1
--max-model-len 128000
--max-num-batched-tokens 128000
--max-num-seqs=16
--trust-remote-code
--enable-chunked-prefill
--enable-prefix-caching --enforce-eager
--tensor-parallel-size 8 --gpu-memory-utilization 0.98
--api-key 0f3b19e46dd3aea61ca94177daf9e609
--enable-auto-tool-choice --tool-call-parser hermes

curl http://localhost:9995/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer 0f3b19e46dd3aea61ca94177daf9e609" \
-N \
-d '{
"model": "deepseek-r1",
"messages": [{"role": "user", "content": "写一首关于春天的诗"}],
"stream": true,
"max_tokens": 256
}'

 

python3 -m vllm.entrypoints.openai.api_server --model /models/Qwen3-235B-A22B-Instruct-2507 --host 0.0.0.0 --port 30001 --tensor-parallel-size 8 --trust-remote-code --distributed-executor-backend=mp --max-model-len 262144 --gpu-memory-utilization 0.85 --served-model-name=Qwen3-235B-A22B-Instruct-2507 --enable-auto-tool-choice --tool-call-parser=hermes --api-key ad0ecdc21b2347eaaef4adba5ee59c37