高性能的UDP服务器的网络层

本章我将谈谈如何开发一个高性能的UDP服务器的网络层.UDP服务器的网络层开发相对与TCP服务器来说要容易和简单的多,UDP服务器的大致流程为创建一个socket然后将其绑定到完成端口上并投递一定数量的recv操作.当有数据到来时从完成队列中取出数据发送到接收队列中即可。
  测试结果如下：
    WindowsXP Professional，Intel Core Duo E4600 双核2.4G , 2G内存。同时30K个用户和该UDP服务器进行交互其CPU使用率为10%左右，内存占用7M左右。
  下面详细介绍该服务器的架构及流程：  
1. 首先介绍服务器的接收和发送缓存UDP_CONTEXT。

class UDP_CONTEXT : protected NET_CONTEXT
{
      friend class UdpSer;
      protected:
      IP_ADDR m_RemoteAddr;            //对端地址

      enum
     {
           HEAP_SIZE = 1024 * 1024 * 5,
           MAX_IDL_DATA = 10000,
      };

    public:
        UDP_CONTEXT() {}
        virtual ~UDP_CONTEXT() {}

       void* operator new(size_t nSize);
       void operator delete(void* p);

    private:
        static vector<UDP_CONTEXT* > s_IDLQue;
        static CRITICAL_SECTION s_IDLQueLock;
        static HANDLE s_hHeap;    
};

　　UDP_CONTEXT的实现流程和TCP_CONTEXT的实现流程大致相同，此处就不进行详细介绍。

2. UDP_RCV_DATA，当服务器收到客户端发来的数据时会将数据以UDP_RCV_DATA的形式放入到数据接收队列中，其声明如下：

class DLLENTRY UDP_RCV_DATA
{
	friend class UdpSer;
public:
	CHAR* m_pData;                //数据缓冲区
	INT m_nLen;                    //数据的长度
	IP_ADDR m_PeerAddr;            //发送报文的地址

	UDP_RCV_DATA(const CHAR* szBuf, int nLen, const IP_ADDR& PeerAddr);
	~UDP_RCV_DATA();

	void* operator new(size_t nSize);
	void operator delete(void* p);

	enum
	{
		RCV_HEAP_SIZE = 1024 * 1024 *50,        //s_Heap堆的大小
		DATA_HEAP_SIZE = 100 * 1024* 1024,    //s_DataHeap堆的大小
		MAX_IDL_DATA = 250000,
	};

private:
	static vector<UDP_RCV_DATA* > s_IDLQue;
	static CRITICAL_SECTION s_IDLQueLock;
	static HANDLE s_DataHeap;        //数据缓冲区的堆
	static HANDLE s_Heap;            //RCV_DATA的堆
};

　　UDP_RCV_DATA的实现和TCP_RCV_DATA大致相同, 此处不在详细介绍.

下面将主要介绍UdpSer类, 该类主要用来管理UDP服务.其定义如下:

class DLLENTRY UdpSer
{
public:
	UdpSer();
	~UdpSer();

	/************************************************************************
	* Desc : 初始化静态资源，在申请UDP实例对象之前应先调用该函数, 否则程序无法正常运行
	************************************************************************/
	static void InitReource();

	/************************************************************************
	* Desc : 在释放UDP实例以后, 掉用该函数释放相关静态资源
	************************************************************************/
	static void ReleaseReource();

	//用指定本地地址和端口进行初始化
	BOOL StartServer(const CHAR* szIp = "0.0.0.0", INT nPort = 0);

	//从数据队列的头部获取一个接收数据, pCount不为null时返回队列的长度
	UDP_RCV_DATA* GetRcvData(DWORD* pCount);

	//向对端发送数据
	BOOL SendData(const IP_ADDR& PeerAddr, const CHAR* szData, INT nLen);

	/****************************************************
	* Name : CloseServer()
	* Desc : 关闭服务器
	****************************************************/
	void CloseServer();

protected:
	SOCKET m_hSock;
	vector<UDP_RCV_DATA* > m_RcvDataQue;                //接收数据队列
	CRITICAL_SECTION m_RcvDataLock;                        //访问m_RcvDataQue的互斥锁
	long volatile m_bThreadRun;                                //是否允许后台线程继续运行
	BOOL m_bSerRun;                                            //服务器是否正在运行

	HANDLE *m_pThreads;                //线程数组
	HANDLE m_hCompletion;                    //完成端口句柄

	void ReadCompletion(BOOL bSuccess, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped);

	/****************************************************
	* Name : WorkThread()
	* Desc : I/O 后台管理线程
	****************************************************/
	static UINT WINAPI WorkThread(LPVOID lpParam);
};

　　1. InitReource() 主要对相关的静态资源进行初始化.其实大致和TcpServer::InitReource()大致相同.在UdpSer实例使用之前必须调用该函数进行静态资源的初始化, 否则服务器无法正常使用.

2.ReleaseReource() 主要对相关静态资源进行释放.只有在应用程序结束时才能调用该函数进行静态资源的释放.

3. StartServer() 
该函数的主要功能启动一个UDP服务.其大致流程为先创建服务器UDP socket, 将其绑定到完成端口上然后投递一定数量的recv操作以接收客户端的数据.其实现如下:

BOOL UdpSer::StartServer(const CHAR* szIp /* =  */, INT nPort /* = 0 */)
{
	BOOL bRet = TRUE;
	const int RECV_COUNT = 500;
	WSABUF RcvBuf = { NULL, 0 };
	DWORD dwBytes = 0;
	DWORD dwFlag = 0;
	INT nAddrLen = sizeof(IP_ADDR);
	INT iErrCode = 0;

	try
	{
		if (m_bSerRun)
		{
			THROW_LINE;
		}

		m_bSerRun = TRUE;
		m_hSock = WSASocket(AF_INET, SOCK_DGRAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
		if (INVALID_SOCKET == m_hSock)
		{
			THROW_LINE;
		}
		ULONG ul = 1;
		ioctlsocket(m_hSock, FIONBIO, &ul);

		//设置为地址重用，优点在于服务器关闭后可以立即启用
		int nOpt = 1;
		setsockopt(m_hSock, SOL_SOCKET, SO_REUSEADDR, (char*)&nOpt, sizeof(nOpt));

		//关闭系统缓存，使用自己的缓存以防止数据的复制操作
		INT nZero = 0;
		setsockopt(m_hSock, SOL_SOCKET, SO_SNDBUF, (char*)&nZero, sizeof(nZero));
		setsockopt(m_hSock, SOL_SOCKET, SO_RCVBUF, (CHAR*)&nZero, sizeof(nZero));

		IP_ADDR addr(szIp, nPort);
		if (SOCKET_ERROR == bind(m_hSock, (sockaddr*)&addr, sizeof(addr)))
		{
			closesocket(m_hSock);
			THROW_LINE;
		}

		//将SOCKET绑定到完成端口上
		CreateIoCompletionPort((HANDLE)m_hSock, m_hCompletion, 0, 0);

		//投递读操作
		for (int nIndex = 0; nIndex < RECV_COUNT; nIndex++)
		{
			UDP_CONTEXT* pRcvContext = new UDP_CONTEXT();
			if (pRcvContext && pRcvContext->m_pBuf)
			{
				dwFlag = 0;
				dwBytes = 0;
				nAddrLen = sizeof(IP_ADDR);
				RcvBuf.buf = pRcvContext->m_pBuf;
				RcvBuf.len = UDP_CONTEXT::S_PAGE_SIZE;

				pRcvContext->m_hSock = m_hSock;
				pRcvContext->m_nOperation = OP_READ;            
				iErrCode = WSARecvFrom(pRcvContext->m_hSock, &RcvBuf, 1, &dwBytes, &dwFlag, (sockaddr*)(&pRcvContext->m_RemoteAddr)
					, &nAddrLen, &(pRcvContext->m_ol), NULL);
				if (SOCKET_ERROR == iErrCode && ERROR_IO_PENDING != WSAGetLastError())
				{
					delete pRcvContext;
					pRcvContext = NULL;
				}
			}
			else
			{
				delete pRcvContext;
			}
		}
	}
	catch (const long &lErrLine)
	{            
		bRet = FALSE;
		_TRACE("Exp : %s -- %ld ", __FILE__, lErrLine);            
	}

	return bRet;
}

　　4. GetRcvData(), 从接收队列中取出一个数据包.

UDP_RCV_DATA *UdpSer::GetRcvData(DWORD* pCount)
{
    UDP_RCV_DATA* pRcvData = NULL;

    EnterCriticalSection(&m_RcvDataLock);
    vector<UDP_RCV_DATA* >::iterator iterRcv = m_RcvDataQue.begin();
    if (iterRcv != m_RcvDataQue.end())
    {
        pRcvData = *iterRcv;
        m_RcvDataQue.erase(iterRcv);
    }

    if (pCount)
    {
        *pCount = (DWORD)(m_RcvDataQue.size());
    }
    LeaveCriticalSection(&m_RcvDataLock);

    return pRcvData;
}

5. SendData() 发送指定长度的数据包.

BOOL UdpSer::SendData(const IP_ADDR& PeerAddr, const CHAR* szData, INT nLen)
{
    BOOL bRet = TRUE;
    try
    {
        if (nLen >= 1500)
        {
            THROW_LINE;
        }

        UDP_CONTEXT* pSendContext = new UDP_CONTEXT();
        if (pSendContext && pSendContext->m_pBuf)
        {
            pSendContext->m_nOperation = OP_WRITE;
            pSendContext->m_RemoteAddr = PeerAddr;        

            memcpy(pSendContext->m_pBuf, szData, nLen);

            WSABUF SendBuf = { NULL, 0 };
            DWORD dwBytes = 0;
            SendBuf.buf = pSendContext->m_pBuf;
            SendBuf.len = nLen;

            INT iErrCode = WSASendTo(m_hSock, &SendBuf, 1, &dwBytes, 0, (sockaddr*)&PeerAddr, sizeof(PeerAddr), &(pSendContext->m_ol), NULL);
            if (SOCKET_ERROR == iErrCode && ERROR_IO_PENDING != WSAGetLastError())
            {
                delete pSendContext;
                THROW_LINE;
            }
        }
        else
        {
            delete pSendContext;
            THROW_LINE;
        }
    }
    catch (const long &lErrLine)
    {
        bRet = FALSE;
        _TRACE("Exp : %s -- %ld ", __FILE__, lErrLine);            
    }

    return bRet;
}

6. CloseServer() 关闭服务

void UdpSer::CloseServer()
{
    m_bSerRun = FALSE;
    closesocket(m_hSock);
}

7. WorkThread() 在完成端口上工作的后台线程

UINT WINAPI UdpSer::WorkThread(LPVOID lpParam)
{
    UdpSer *pThis = (UdpSer *)lpParam;
    DWORD dwTrans = 0, dwKey = 0;
    LPOVERLAPPED pOl = NULL;
    UDP_CONTEXT *pContext = NULL;

    while (TRUE)
    {
        BOOL bOk = GetQueuedCompletionStatus(pThis->m_hCompletion, &dwTrans, &dwKey, (LPOVERLAPPED *)&pOl, WSA_INFINITE);

        pContext = CONTAINING_RECORD(pOl, UDP_CONTEXT, m_ol);
        if (pContext)
        {
            switch (pContext->m_nOperation)
            {
            case OP_READ:
                pThis->ReadCompletion(bOk, dwTrans, pOl);
                break;
            case OP_WRITE:
                delete pContext;
                pContext = NULL;
                break;
            }
        }

        if (FALSE == InterlockedExchangeAdd(&(pThis->m_bThreadRun), 0))
        {
            break;
        }
    }

    return 0;
}

8.ReadCompletion(), 接收操作完成后的回调函数

void UdpSer::ReadCompletion(BOOL bSuccess, DWORD dwNumberOfBytesTransfered, LPOVERLAPPED lpOverlapped)
{
    UDP_CONTEXT* pRcvContext = CONTAINING_RECORD(lpOverlapped, UDP_CONTEXT, m_ol);
    WSABUF RcvBuf = { NULL, 0 };
    DWORD dwBytes = 0;
    DWORD dwFlag = 0;
    INT nAddrLen = sizeof(IP_ADDR);
    INT iErrCode = 0;

    if (TRUE == bSuccess && dwNumberOfBytesTransfered <= UDP_CONTEXT::S_PAGE_SIZE)
    {
#ifdef _XML_NET_
        EnterCriticalSection(&m_RcvDataLock);

        UDP_RCV_DATA* pRcvData = new UDP_RCV_DATA(pRcvContext->m_pBuf, dwNumberOfBytesTransfered, pRcvContext->m_RemoteAddr);
        if (pRcvData && pRcvData->m_pData)
        {
            m_RcvDataQue.push_back(pRcvData);
        }    
        else
        {
            delete pRcvData;
        }

        LeaveCriticalSection(&m_RcvDataLock);
#else
        if (dwNumberOfBytesTransfered >= sizeof(PACKET_HEAD))
        {
            EnterCriticalSection(&m_RcvDataLock);

            UDP_RCV_DATA* pRcvData = new UDP_RCV_DATA(pRcvContext->m_pBuf, dwNumberOfBytesTransfered, pRcvContext->m_RemoteAddr);
            if (pRcvData && pRcvData->m_pData)
            {
                m_RcvDataQue.push_back(pRcvData);
            }    
            else
            {
                delete pRcvData;
            }

            LeaveCriticalSection(&m_RcvDataLock);
        }
#endif

        //投递下一个接收操作
        RcvBuf.buf = pRcvContext->m_pBuf;
        RcvBuf.len = UDP_CONTEXT::S_PAGE_SIZE;

        iErrCode = WSARecvFrom(pRcvContext->m_hSock, &RcvBuf, 1, &dwBytes, &dwFlag, (sockaddr*)(&pRcvContext->m_RemoteAddr)
            , &nAddrLen, &(pRcvContext->m_ol), NULL);
        if (SOCKET_ERROR == iErrCode && ERROR_IO_PENDING != WSAGetLastError())
        {
            ATLTRACE("\r\n%s -- %ld dwNumberOfBytesTransfered = %ld, LAST_ERR = %ld"
                , __FILE__, __LINE__, dwNumberOfBytesTransfered, WSAGetLastError());
            delete pRcvContext;
            pRcvContext = NULL;
        }
    }
    else
    {
        delete pRcvContext;
    }
}