linux epoll 任务队列多线程模型

/*
*
*EPOLL ET 触发必须使用非阻塞，LT触发可以阻塞/非阻塞。
*read 函数 非阻塞读需 忙轮寻 soket关闭返回0，循环读完数据
*如果已经读完再读read返回 -1，errno=11(EAGIAN)则退出轮循
*
**/


#include<stdio.h>
#include<sys/socket.h>
#include<sys/types.h>
#include<stdlib.h>
#include<unistd.h>
#include<sys/select.h>
#include<sys/time.h>
#include<pthread.h>
#include<memory.h>
#include<errno.h>
#include<arpa/inet.h>
#include<netinet/in.h>
#include<signal.h>
#include<semaphore.h>
#include<malloc.h>
#include<fcntl.h>
#include<sys/epoll.h>
#include<sys/ipc.h>
#include<sys/msg.h>
#define  WKEY 100
#define  RKEY 101
//#define  DEBUG

typedef unsigned int uint32;

pthread_mutex_t lock;
uint32 qcount=0;

typedef struct{
    int quenu;
    sem_t sem;
}QUE_SEM;

QUE_SEM rque ,wque;

typedef struct{
    int flag;
    sem_t sem;
}MYLOCK;

typedef  struct{
    long type;
    char buff[1024];
}REQUEST;

typedef  struct{
    long type;
    int wrfd;
}RESPON;

struct sockaddr_in server;
MYLOCK mlock;
int g_epfd=-1;

int rnum=0;
int wnum=0;

void printerror()
{
    printf("fun %s,%d num",__func__,__LINE__);
     printf("rnum =%d ,wnum =%d\n",rnum,wnum);
    printf("%d--->%s\n",errno,strerror(errno));
    msgctl(rque.quenu,IPC_RMID,0);
    msgctl(wque.quenu,IPC_RMID,0);
    exit(-1);
}

void initREQUESTue(int rkey,int wkey){
  int rq= msgget(rkey,IPC_CREAT|0666);
  int wq= msgget(wkey,IPC_CREAT|0666);
  sem_init(&rque.sem,0,1);
  sem_init(&wque.sem,0,1);
  rque.quenu=rq,wque.quenu=wq;

  printf("REQUEST r is %d ,w is %d\n",rq,wq);
}

int ReadSocket(int fd){
        REQUEST msg;
        int rd=-1;
        memset(msg.buff,0,sizeof (msg.buff));
        msg.type=fd;
        rd=read(fd,msg.buff,sizeof(msg.buff)-1);
//               printf("rd size is %d\n",rd);
        if(rd==0){
            struct sockaddr_in client;
            memset(&client,0,sizeof(client));
            int len=sizeof(client);
            getpeername(fd,(struct sockaddr*)&client,&len);
//            printf("client %s is closed fd is %d , bye!\n",inet_ntoa(client.sin_addr),fd);
            close(fd);
            return 0;
        }else if(rd>0){
            int res=-1;
            while(1){
               sem_wait(&rque.sem);
               res =msgsnd(rque.quenu,&msg,sizeof(REQUEST)-sizeof(long),IPC_NOWAIT);
                    printf("read clietn and send to queue res=%d\n",res);
               if(res==-1){
                        printf("errno =%d msg=%s\n",errno,strerror(errno));
                   if(errno==ENOMEM){
                       sem_post(&rque.sem);
                       usleep(100);
                       continue;
                   }
                   sem_post(&rque.sem);
                   return 0;
               }else if(res==0){
//                    printf("rnum=%d\n",rnum++);
                   sem_post(&rque.sem);
                   return 1;
               }
            }

        }else if(rd==-1){
            close(fd);
            return 0;
        }
}


//int getMsgbyfd(int fd,REQUEST* msg){
//   if(msgrcv(rque.quenu,msg,sizeof(REQUEST)-sizeof(long),fd,0)>0){
//       return 1;
//   }
//   return 0;
//}

void* th_procce(void* p){
    while(1){
        sem_wait(&wque.sem);
        RESPON msg;
            #ifdef DEBUG
            printf("proc thread ready to feth msg from wque\n");
            #endif
        int res=msgrcv(wque.quenu,&msg,sizeof(RESPON)-sizeof(long),0,IPC_NOWAIT);
            usleep(3000);
            #ifdef DEBUG
                    printf("proc feth msg from wque type=%d\n",res);
            #endif
        sem_post(&wque.sem);
        usleep(100);
        if(res>0){
             sem_wait(&rque.sem);
                 REQUEST req;
              int  res2=msgrcv(rque.quenu,&req,sizeof(REQUEST)-sizeof(long),msg.wrfd,IPC_NOWAIT);
            #ifdef DEBUG
                    printf("proc feth msg from rque and send to client res=%d\n",res2);
            #endif
                 if(res2>0){
             sem_post(&rque.sem);
                     char buff[1024]={0};
                     sprintf(buff,"%s -> %s","Server Snd To",req.buff);
                     write(msg.wrfd,buff,strlen(buff));
                 }else if(res2==-1&& errno==ENOMSG){
             sem_post(&rque.sem);
                     continue;
                 }
             usleep(100);
            #ifdef DEBUG
                    printf("yet send to client\n");
            #endif
        }else if(res==-1&& errno==ENOMSG){
            continue;
        }
    }
}

int sndtowque(int fd){
    RESPON msg;
    msg.type=fd;
    msg.wrfd=fd;
    while (1) {
        sem_wait(&wque.sem);
            int res=msgsnd(wque.quenu,&msg,sizeof(RESPON)-sizeof(long),IPC_NOWAIT);
//            #ifdef DEBUG
                    printf("write event com and sendto wque res=%d\n",res);
//            #endif
            if(res==-1){
                if(errno==ENOMEM){
                    sem_post(&wque.sem);
                    usleep(200);
                    continue;
                }
                sem_post(&wque.sem);
                return 0;
            }else if(res==0){
                    printf("wnum-----=%d\n",wnum++);
                sem_post(&wque.sem);
                return 1;
            }
    }
}

void* th_hand(void* p){
    sem_wait(&mlock.sem);
    while(!mlock.flag){
        sem_post(&mlock.sem);
        sleep(1);
        sem_wait(&mlock.sem);
    }
    sem_post(&mlock.sem);
    struct epoll_event events[10];
    struct epoll_event event;
    while(1){
        memset(&event,0,sizeof(struct epoll_event));
        memset(events,0,sizeof(struct epoll_event)*10);
        int s = epoll_wait(g_epfd,events,10,100);
        if(s==-1){
            printerror();
        }else if(s==0){
            continue;
        }else if(s>0){
            for(int i=0;i<s;i++){
                int fd=events[i].data.fd;
                if(events[i].events & EPOLLIN){
                    if(ReadSocket(fd)){
                        events[i].events=EPOLLOUT|EPOLLET;
                        epoll_ctl(g_epfd,EPOLL_CTL_MOD,fd,&events[i]);
                    }else{
                        epoll_ctl(g_epfd,EPOLL_CTL_DEL,fd,0);
                        continue;
                    }
                }
                if(events[i].events & EPOLLOUT){
                    if(sndtowque(fd)){
                        events[i].events=EPOLLIN|EPOLLET;
                        epoll_ctl(g_epfd,EPOLL_CTL_MOD,events[i].data.fd,&events[i]);
                    }else{
                        epoll_ctl(g_epfd,EPOLL_CTL_DEL,fd,0);
                        continue;
                    }
                }
            }
        }
    }
}


int initSocket(int port){
    memset(&server,0,sizeof(server));
    server.sin_family=AF_INET;
    server.sin_addr.s_addr=htonl(INADDR_ANY);//INADDR_ANY==0
    server.sin_port=htons(port);
    int sockfd=socket(AF_INET,SOCK_STREAM,0);
        int flag=1;
     setsockopt(sockfd,SOL_SOCKET,SO_REUSEADDR,&flag,sizeof(flag));
    if(sockfd==-1){
        printerror();
    }
    int res=bind(sockfd,(struct sockaddr*)&server,sizeof(struct sockaddr));
    if(res==-1){
        printerror();
    }
    if(-1==listen(sockfd,10)){
        printerror();
    }

    g_epfd=epoll_create(5);
    if(g_epfd==-1){
        printerror();
    }

    sem_wait(&mlock.sem);
        mlock.flag=1;
    sem_post(&mlock.sem);

    printf("main before accept\n");
    while(1){
        int fd;
        if((fd=accept(sockfd,NULL,NULL))==-1){
            printerror();
        }
        printf("fd %d is connect\n",fd);
        struct epoll_event event;
//        event.events=EPOLLIN;
        event.events=EPOLLIN|EPOLLET;
        event.data.fd=fd;

        fcntl(fd,F_SETFL,fcntl(fd,F_GETFL,0)|O_NONBLOCK);

        sem_wait(&mlock.sem);
        if(-1==(epoll_ctl(g_epfd,EPOLL_CTL_ADD,fd,&event))){
            printerror();
        }
        qcount++;
        sem_post(&mlock.sem);
    }
}

void sig_hand(int signo){
    if(signo==SIGINT){
        msgctl(rque.quenu,IPC_RMID,0);
        msgctl(wque.quenu,IPC_RMID,0);
        printf("have %d client\n",qcount);
        exit(0);
    }
}

pthread_t pid;
pthread_t process;
int main(int argc,char** argv){
    if(argc<2){
        puts("please input port\n");
        exit(-1);
    }
    initREQUESTue(RKEY,WKEY);
    int port=atoi(argv[1]);
    signal(SIGINT,sig_hand);
    memset(&mlock,0,sizeof(mlock));
    sem_init(&mlock.sem,0,1);
    mlock.flag=0;

    pthread_create(&pid,NULL,th_hand,(void*)0);
    pthread_detach(pid);

    pthread_create(&process,NULL,th_procce,(void*)0);
    pthread_detach(process);
    initSocket(port);

}