bufferevent 与 socket
http://blog.sina.com.cn/s/blog_56dee71a0100qx4s.html
很多时候，除了响应事件之外，应用还希望做一定的数据缓冲。比如说，写入数据的时候，通常的运行模式是：
l 决定要向连接写入一些数据，把数据放入到缓冲区中
l 等待连接可以写入
l 写入尽量多的数据
l 记住写入了多少数据，如果还有更多数据要写入，等待连接再次可以写入
这种缓冲IO模式很通用，libevent为此提供了一种通用机制，即bufferevent。bufferevent由一个底层的传输端口（如套接字），一个读取缓冲区和一个写入缓冲区组成。与通常的事件在底层传输端口已经就绪，可以读取或者写入的时候执行回调不同的是，bufferevent在读取或者写入了足够量的数据之后调用用户提供的回调。
bufferevent 的简单范例
这里选取了 Libevent 的一个范例程序 hello-world.c 来看看 Libevent 的用法：
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <signal.h>
#ifndef WIN32
#include <netinet/in.h>
# ifdef _XOPEN_SOURCE_EXTENDED
# include <arpa/inet.h>
# endif
#include <sys/socket.h>
#endif
// bufferevent
#include <event2/bufferevent.h>
// bufferevent 使用的 buffer
#include <event2/buffer.h>
// 连接监听器
#include <event2/listener.h>
#include <event2/util.h>
#include <event2/event.h>
static const char MESSAGE[] = "Hello, World!\n";
static const int PORT = 9995;
// 新连接到来时的回调
static void listener_cb(struct evconnlistener *, evutil_socket_t,
struct sockaddr *, int socklen, void *);
// 读取回调
static void conn_writecb(struct bufferevent *, void *);
// 事件回调
static void conn_eventcb(struct bufferevent *, short, void *);
// 信号回调
static void signal_cb(evutil_socket_t, short, void *);
int
main(int argc, char **argv)
{
struct event_base *base;
struct evconnlistener *listener;
struct event *signal_event;
struct sockaddr_in sin;
#ifdef WIN32
WSADATA wsa_data;
WSAStartup(0x0201, &wsa_data);
#endif
// 首先构建 base
base = event_base_new();
if (!base) {
fprintf(stderr, "Could not initialize libevent!\n");
return 1;
}
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_port = htons(PORT);
// 创建监听器
listener = evconnlistener_new_bind(base, listener_cb, (void *)base,
LEV_OPT_REUSEABLE|LEV_OPT_CLOSE_ON_FREE, -1,
(struct sockaddr*)&sin,
sizeof(sin));
if (!listener) {
fprintf(stderr, "Could not create a listener!\n");
return 1;
}
// 中断信号
signal_event = evsignal_new(base, SIGINT, signal_cb, (void *)base);
if (!signal_event || event_add(signal_event, NULL)<0) {
fprintf(stderr, "Could not create/add a signal event!\n");
return 1;
}
event_base_dispatch(base);
evconnlistener_free(listener);
event_free(signal_event);
event_base_free(base);
printf("done\n");
return 0;
}
static void
listener_cb(struct evconnlistener *listener, evutil_socket_t fd,
struct sockaddr *sa, int socklen, void *user_data)
{
struct event_base *base = user_data;
struct bufferevent *bev;
// 得到一个新的连接，通过连接 fd 构建一个 bufferevent
bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);
if (!bev) {
fprintf(stderr, "Error constructing bufferevent!");
event_base_loopbreak(base);
return;
}
// 设置创建的 bufferevent 的回调函数
bufferevent_setcb(bev, NULL, conn_writecb, conn_eventcb, NULL);
bufferevent_enable(bev, EV_WRITE);
bufferevent_disable(bev, EV_READ);
// 写入数据到 bufferevent 中
bufferevent_write(bev, MESSAGE, strlen(MESSAGE));
}
static void
conn_writecb(struct bufferevent *bev, void *user_data)
{
struct evbuffer *output = bufferevent_get_output(bev);
if (evbuffer_get_length(output) == 0) {
printf("flushed answer\n");
bufferevent_free(bev);
}
}
static void
conn_eventcb(struct bufferevent *bev, short events, void *user_data)
{
if (events & BEV_EVENT_EOF) {
printf("Connection closed.\n");
} else if (events & BEV_EVENT_ERROR) {
printf("Got an error on the connection: %s\n",
strerror(errno));/*XXX win32*/
}
/* None of the other events can happen here, since we haven't enabled
* timeouts */
bufferevent_free(bev);
}
static void
signal_cb(evutil_socket_t sig, short events, void *user_data)
{
struct event_base *base = user_data;
struct timeval delay = { 2, 0 };
printf("Caught an interrupt signal; exiting cleanly in two seconds.\n");
// 停止事件循环
event_base_loopexit(base, &delay);
}
研究 bufferevent 的关键代码
这里只研究基于 socket 的 bufferevent。从上面 bufferevent 的使用可以看出，有几个关键函数：
- 开始需要调用 bufferevent_socket_new 创建一个 bufferevent
- 调用 bufferevent_setcb 设置回调函数
- 调用 bufferevent_write 写入数据
- 调用 bufferevent_free 释放 bufferevent
bufferevent_socket_new 的源码以及分析如下：
// base --- 新创建的 bufferevent 关联的 base
// fd --- bufferevent 关联的文件描述符
struct bufferevent *
bufferevent_socket_new(struct event_base *base, evutil_socket_t fd,
int options)
{
// bufferevent_private 结构体持有 bufferevent 的数据
struct bufferevent_private *bufev_p;
// bufev == &(bufev_p->bev);
// struct bufferevent 中存放的是不同类型的 bufferevent 所共有的部分
// struct bufferevent 是 struct bufferevent_private 的子集
struct bufferevent *bufev;
// windows 下如果启用 IOCP 则构建异步 IO bufferevent
#ifdef WIN32
if (base && event_base_get_iocp(base))
// 细节略
return bufferevent_async_new(base, fd, options);
#endif
if ((bufev_p = mm_calloc(1, sizeof(struct bufferevent_private)))== NULL)
return NULL;
// 初始化 bufferevent_private
// 由于 bufferevent 有不同类型，所以这里设计了 bufferevent_ops_socket
// 对于不同类型的 bufferevent 有不同的 bufferevent_ops_socket 对象
// bufferevent_ops_socket 包括函数指针和一些信息
if (bufferevent_init_common(bufev_p, base, &bufferevent_ops_socket,
options) < 0) {
mm_free(bufev_p);
return NULL;
}
bufev = &bufev_p->bev;
// 设置 EVBUFFER_FLAG_DRAINS_TO_FD，此选项和 evbuffer_add_file() 函数有关（详见文档）
evbuffer_set_flags(bufev->output, EVBUFFER_FLAG_DRAINS_TO_FD);
// 初始化 read 和 write event
// 一个 bufferevent（一个 fd）关联两个 event 对象 ev_read 和 ev_write
// ev_read --- socket 可读或者超时
// ev_write --- socket 可写或者超时
// 它们都未使用 Edge triggered 方式
event_assign(&bufev->ev_read, bufev->ev_base, fd,
EV_READ|EV_PERSIST, bufferevent_readcb, bufev);
event_assign(&bufev->ev_write, bufev->ev_base, fd,
EV_WRITE|EV_PERSIST, bufferevent_writecb, bufev);
// 为输出缓冲区设置回调
// 当输出缓冲区被修改时调用 bufferevent_socket_outbuf_cb 回调函数
evbuffer_add_cb(bufev->output, bufferevent_socket_outbuf_cb, bufev);
// 防止输入缓冲区和输出缓冲区被意外修改
evbuffer_freeze(bufev->input, 0);
evbuffer_freeze(bufev->output, 1);
return bufev;
}
其中 bufferevent_init_common 函数实现为：
int
bufferevent_init_common(struct bufferevent_private *bufev_private,
struct event_base *base,
const struct bufferevent_ops *ops,
enum bufferevent_options options)
{
struct bufferevent *bufev = &bufev_private->bev;
// 创建输入缓冲区
if (!bufev->input) {
if ((bufev->input = evbuffer_new()) == NULL)
return -1;
}
// 创建输出缓冲区
if (!bufev->output) {
if ((bufev->output = evbuffer_new()) == NULL) {
evbuffer_free(bufev->input);
return -1;
}
}
// 初始化 bufferevent 的引用计数
bufev_private->refcnt = 1;
bufev->ev_base = base;
/* Disable timeouts. */
// 清理超时时间
evutil_timerclear(&bufev->timeout_read);
evutil_timerclear(&bufev->timeout_write);
bufev->be_ops = ops;
/*
* Set to EV_WRITE so that using bufferevent_write is going to
* trigger a callback. Reading needs to be explicitly enabled
* because otherwise no data will be available.
*/
// enabled 被 bufferevent_get_enabled 函数返回
// enabled 的值可以为 EV_WRITE EV_READ
bufev->enabled = EV_WRITE;
// bufferevent 相关线程初始化
#ifndef _EVENT_DISABLE_THREAD_SUPPORT
if (options & BEV_OPT_THREADSAFE) {
if (bufferevent_enable_locking(bufev, NULL) < 0) {
/* cleanup */
evbuffer_free(bufev->input);
evbuffer_free(bufev->output);
bufev->input = NULL;
bufev->output = NULL;
return -1;
}
}
#endif
// 选项正确性检查
if ((options & (BEV_OPT_DEFER_CALLBACKS|BEV_OPT_UNLOCK_CALLBACKS))
== BEV_OPT_UNLOCK_CALLBACKS) {
event_warnx("UNLOCK_CALLBACKS requires DEFER_CALLBACKS");
return -1;
}
// defer callbacks 初始化
if (options & BEV_OPT_DEFER_CALLBACKS) {
if (options & BEV_OPT_UNLOCK_CALLBACKS)
event_deferred_cb_init(&bufev_private->deferred,
bufferevent_run_deferred_callbacks_unlocked,
bufev_private);
else
event_deferred_cb_init(&bufev_private->deferred,
bufferevent_run_deferred_callbacks_locked,
bufev_private);
}
bufev_private->options = options;
// 关联 bufferevent 和 buffer
evbuffer_set_parent(bufev->input, bufev);
evbuffer_set_parent(bufev->output, bufev);
return 0;
}
bufferevent 创建成功之后，fd 上存在数据可读则调用 bufferevent_readcb
// fd 可读
static void
bufferevent_readcb(evutil_socket_t fd, short event, void *arg)
{
struct bufferevent *bufev = arg;
struct bufferevent_private *bufev_p =
EVUTIL_UPCAST(bufev, struct bufferevent_private, bev);
struct evbuffer *input;
int res = 0;
short what = BEV_EVENT_READING;
ev_ssize_t howmuch = -1, readmax=-1;
_bufferevent_incref_and_lock(bufev);
// 如果超时了
if (event == EV_TIMEOUT) {
/* Note that we only check for event==EV_TIMEOUT. If
* event==EV_TIMEOUT|EV_READ, we can safely ignore the
* timeout, since a read has occurred */
what |= BEV_EVENT_TIMEOUT;
goto error;
}
input = bufev->input;
/*
* If we have a high watermark configured then we don't want to
* read more data than would make us reach the watermark.
*/
// 是否设置了输入缓冲区的最大大小
if (bufev->wm_read.high != 0) {
howmuch = bufev->wm_read.high - evbuffer_get_length(input);
/* we somehow lowered the watermark, stop reading */
// 缓冲区中数据过多
if (howmuch <= 0) {
// 暂停 bufferevent 的数据读取
// 具体的做法是移除 read event（ev_read）
bufferevent_wm_suspend_read(bufev);
goto done;
}
}
// 获取可读最大大小
// 和限速有关，如果不限速，则为 MAX_TO_READ_EVER(16384) 也就是 16K
readmax = _bufferevent_get_read_max(bufev_p);
if (howmuch < 0 || howmuch > readmax) /* The use of -1 for "unlimited"
* uglifies this code. XXXX */
howmuch = readmax;
// 如果读取暂停
if (bufev_p->read_suspended)
goto done;
// 输入缓冲区可读
evbuffer_unfreeze(input, 0);
// 读取 fd 上的数据
res = evbuffer_read(input, fd, (int)howmuch); /* XXXX evbuffer_read would do better to take and return ev_ssize_t */
// 输入缓冲区禁止读取
evbuffer_freeze(input, 0);
// 读取数据失败
if (res == -1) {
// 获取到错误
int err = evutil_socket_geterror(fd);
// EINTR、EAGAIN
// Windows 下为 WSAEWOULDBLOCK、WSAEINTR
if (EVUTIL_ERR_RW_RETRIABLE(err))
goto reschedule;
// 如果错误是不可重试的，报错
/* error case */
what |= BEV_EVENT_ERROR;
// eof
} else if (res == 0) {
/* eof case */
what |= BEV_EVENT_EOF;
}
if (res <= 0)
goto error;
_bufferevent_decrement_read_buckets(bufev_p, res);
/* Invoke the user callback - must always be called last */
// 判断是否需要调用回调
if (evbuffer_get_length(input) >= bufev->wm_read.low)
_bufferevent_run_readcb(bufev);
goto done;
reschedule:
goto done;
error:
// 出错后暂停读取数据
bufferevent_disable(bufev, EV_READ);
// 通过事件回调通知出错
_bufferevent_run_eventcb(bufev, what);
done:
_bufferevent_decref_and_unlock(bufev);
}
这里比较关键的函数为 evbuffer_read：
int
evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch)
{
struct evbuffer_chain **chainp;
int n;
int result;
#ifdef USE_IOVEC_IMPL
int nvecs, i, remaining;
#else
struct evbuffer_chain *chain;
unsigned char *p;
#endif
EVBUFFER_LOCK(buf);
// buffer 是否可读
if (buf->freeze_end) {
result = -1;
goto done;
}
// 获取当前 socket 可读的字节数
n = get_n_bytes_readable_on_socket(fd);
if (n <= 0 || n > EVBUFFER_MAX_READ)
n = EVBUFFER_MAX_READ;
// 尽可能多的读取
if (howmuch < 0 || howmuch > n)
howmuch = n;
// 一种实现
#ifdef USE_IOVEC_IMPL
/* Since we can use iovecs, we're willing to use the last
* NUM_READ_IOVEC chains. */
// 调整缓冲区（细节略）
if (_evbuffer_expand_fast(buf, howmuch, NUM_READ_IOVEC) == -1) {
result = -1;
goto done;
} else {
IOV_TYPE vecs[NUM_READ_IOVEC];
#ifdef _EVBUFFER_IOVEC_IS_NATIVE
nvecs = _evbuffer_read_setup_vecs(buf, howmuch, vecs,
NUM_READ_IOVEC, &chainp, 1);
#else
/* We aren't using the native struct iovec. Therefore,
we are on win32. */
struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC];
nvecs = _evbuffer_read_setup_vecs(buf, howmuch, ev_vecs, 2,
&chainp, 1);
for (i=0; i < nvecs; ++i)
WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]);
#endif
#ifdef WIN32
{
// 读取到的数据字节数
DWORD bytesRead;
DWORD flags=0;
// windows 下进行 recv
if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) {
/* The read failed. It might be a close,
* or it might be an error. */
// 这里使用 WSARecv 时需要注意 WSAECONNABORTED 表示了连接关闭了
if (WSAGetLastError() == WSAECONNABORTED)
n = 0;
else
n = -1;
} else
n = bytesRead;
}
#else
// 非 windows 平台 read
n = readv(fd, vecs, nvecs);
#endif
}
// 使用另外一种实现
#else /*!USE_IOVEC_IMPL*/
/* If we don't have FIONREAD, we might waste some space here */
/* XXX we _will_ waste some space here if there is any space left
* over on buf->last. */
if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) {
result = -1;
goto done;
}
/* We can append new data at this point */
p = chain->buffer + chain->misalign + chain->off;
// read
#ifndef WIN32
n = read(fd, p, howmuch);
#else
n = recv(fd, p, howmuch, 0);
#endif
#endif /* USE_IOVEC_IMPL */
if (n == -1) {
result = -1;
goto done;
}
if (n == 0) {
result = 0;
goto done;
}
#ifdef USE_IOVEC_IMPL
remaining = n;
for (i=0; i < nvecs; ++i) {
ev_ssize_t space = (ev_ssize_t) CHAIN_SPACE_LEN(*chainp);
if (space < remaining) {
(*chainp)->off += space;