配置Nginx反向代理

 

 

vllm部署多个模型，可以用nginx做反向代理，负载均衡。

配置如下：

worker_processes auto;
error_log /var/log/nginx/error.log warn;

events {
    worker_connections 1024;
}

http {
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /var/log/nginx/access.log main;

    upstream vllm_backend {
        server 127.0.0.1:8001 max_fails=3 fail_timeout=30s;
        server 127.0.0.1:8002 max_fails=3 fail_timeout=30s;       
        keepalive 32;
    }

    server {
        listen 8005;
        server_name api.example.com;

        proxy_buffering off;
        proxy_read_timeout 600s;
        proxy_send_timeout 300s;
        proxy_connect_timeout 30s;
        chunked_transfer_encoding on;

        location /v1/ {
            proxy_pass http://vllm_backend;   # 去掉末尾斜杠
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }

        location /healthz {
            return 200 'ok';
            add_header Content-Type text/plain;
        }
    }
}