服务器三：多线程epoll

#include <fcntl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <signal.h>
#include <fcntl.h>
#include <sys/wait.h>
#include <sys/epoll.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <vector>
#include <algorithm>
#include <iostream>
#include <pthread.h>
#include <sys/types.h>//包含pthread_self
#include <unistd.h>//包含syscall
#include <sys/syscall.h>

#define gettid() syscall(__NR_gettid)


typedef std::vector<struct epoll_event> EventList;

#define ERR_EXIT(m) \
    do \
{ \
    perror(m); \
    exit(EXIT_FAILURE); \
    } while(0)

void* handleMessage(void* para)
{
    char recvBuf[1024] = {0};
    int ret = 999;
    int socketfd = *(int *)para;

    while(true)
    {
        ret = read(socketfd, recvBuf, 1024);// 接受客户端消息
        if(ret < 0)
        {
            break;// 这里表示读取数据出问题.
            //由于是非阻塞的模式,所以当errno为EAGAIN时,表示当前缓冲区已无数据可//读在这里就当作是该次事件已处理过。
            if(errno == EAGAIN)
            {
                printf("EAGAIN\n");
                break;
            }
            else
            {
                printf("read error! errno:%d\n", errno);
                ret = 0;
                break;
            }
        }
        else if(ret == 0)
        {
            // 这里表示对端的socket已正常关闭.
            break;
        }
        else
        {
            pthread_t tid1 = pthread_self();//用户态ID, 与pthread_create创建出来的相同
            long int tid2 = gettid();//内核态ID
            std::cout << tid1 << " recive data: " << recvBuf;
            std::cout << tid2 << " recive data: " << recvBuf << std::endl;
            write(socketfd, recvBuf, strlen(recvBuf));
            break;
        }
    }
    if(ret == 0)
        return para;
    return NULL;
}

int main(void)
{
    //TCP是全双工的信道, 可以看作两条单工信道, TCP连接两端的两个端点各负责一条. 当对端调用close时, 虽然本意是关闭整个两条信道,
    //但本端只是收到FIN包. 按照TCP协议的语义, 表示对端只是关闭了其所负责的那一条单工信道, 仍然可以继续接收数据. 也就是说, 因为TCP协议的限制,
    //一个端点无法获知对端的socket是调用了close还是shutdown.
    //对一个已经收到FIN包的socket调用read方法,
    //如果接收缓冲已空, 则返回0, 这就是常说的表示连接关闭. 但第一次对其调用write方法时, 如果发送缓冲没问题, 会返回正确写入(发送).
    //但发送的报文会导致对端发送RST报文, 因为对端的socket已经调用了close, 完全关闭, 既不发送, 也不接收数据. 所以,
    //第二次调用write方法(假设在收到RST之后), 会生成SIGPIPE信号, 导致进程退出.
    //为了避免进程退出, 可以捕获SIGPIPE信号, 或者忽略它, 给它设置SIG_IGN信号处理函数:
    //这样, 第二次调用write方法时, 会返回-1, 同时errno置为SIGPIPE. 程序便能知道对端已经关闭.
    signal(SIGPIPE, SIG_IGN);//防止进程退出
    //忽略SIGCHLD信号，这常用于并发服务器的性能的一个技巧
    //因为并发服务器常常fork很多子进程，子进程终结之后需要
    //服务器进程去wait清理资源。如果将此信号的处理方式设为
    //忽略，可让内核把僵尸子进程转交给init进程去处理，省去了
    //大量僵尸进程占用系统资源。(Linux Only)
    signal(SIGCHLD, SIG_IGN);

    int idlefd = open("/dev/null", O_RDONLY | O_CLOEXEC);
    int listenfd;
    //if ((listenfd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
    if ((listenfd = socket(PF_INET, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, IPPROTO_TCP)) < 0)
        ERR_EXIT("socket");

    struct sockaddr_in servaddr;
    memset(&servaddr, 0, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(5188);
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);

    int on = 1;
    if (setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0)
        ERR_EXIT("setsockopt");

    if (bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) < 0)
        ERR_EXIT("bind");
    if (listen(listenfd, SOMAXCONN) < 0)
        ERR_EXIT("listen");

    //create epoll object
    int epollfd;
    epollfd = epoll_create1(EPOLL_CLOEXEC);

    //add EPOLLIN event to epoll
    struct epoll_event event;
    event.data.fd = listenfd;
    event.events = EPOLLIN | EPOLLET;
    epoll_ctl(epollfd, EPOLL_CTL_ADD, listenfd, &event);

    EventList events(16);
    struct sockaddr_in peeraddr;
    socklen_t peerlen;
    int connfd;

    std::vector<int> clients;

    int nready;
    while (1)
    {
        //return active event
        nready = epoll_wait(epollfd, &*events.begin(), static_cast<int>(events.size()), -1);
        if (nready == -1)
        {
            if (errno == EINTR)
                continue;

            ERR_EXIT("epoll_wait");
        }
        if (nready == 0)    // nothing happended
            continue;

        //double capacity
        if ((size_t)nready == events.size())
            events.resize(events.size()*2);

        //treat all active event
        for (int i = 0; i < nready; ++i)
        {
            //如果是主socket的事件的话，则表示有新连接进入了，进行新连接的处理
            if (events[i].data.fd == listenfd)
            {
                peerlen = sizeof(peeraddr);
                connfd = ::accept4(listenfd, (struct sockaddr*)&peeraddr,
                                   &peerlen, SOCK_NONBLOCK | SOCK_CLOEXEC);

                if (connfd == -1)
                {
                    //防止文件句柄超过最大数量
                    if (errno == EMFILE)
                    {
                        close(idlefd);
                        idlefd = accept(listenfd, NULL, NULL);
                        close(idlefd);
                        idlefd = open("/dev/null", O_RDONLY | O_CLOEXEC);
                        continue;
                    }
                    else
                        ERR_EXIT("accept4");
                }


                std::cout<<"ip="<<inet_ntoa(peeraddr.sin_addr)<<
                           " port="<<ntohs(peeraddr.sin_port)<<std::endl;

                clients.push_back(connfd);

                event.data.fd = connfd;
                event.events = EPOLLIN | EPOLLET;
                epoll_ctl(epollfd, EPOLL_CTL_ADD, connfd, &event);
            }
            else if (events[i].events & EPOLLIN)
            {
                connfd = events[i].data.fd;
                if (connfd < 0)
                    continue;

                pthread_attr_t attr;
                pthread_t threadId;

                /*初始化属性值，均设为默认值, 把线程设置为系统级线程*/
                pthread_attr_init(&attr);
                pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
                /* 设置线程为分离属性*/
                //pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);

                if(pthread_create(&threadId, &attr, handleMessage, (void*)&(events[i].data.fd)))
                {
                    perror("pthread_creat error!");
                    exit(-1);
                }
                std::cout << "Create thread: " << threadId << std::endl;

                void* rmpfd = NULL;
                pthread_join(threadId, &rmpfd);

                if(rmpfd != NULL)
                {
                    int rmfd = *(int*)rmpfd;
                    std::cout<<"client close"<<std::endl;
                    close(rmfd);
                    event = events[i];
                    epoll_ctl(epollfd, EPOLL_CTL_DEL, rmfd, &event);
                    clients.erase(std::remove(clients.begin(), clients.end(), rmfd), clients.end());
                    continue;
                }
            }

        }
    }

    return 0;
}