协程--eventlet

一 greenlet

　　greenlet使用c语言实现的Python模块，使Python支持协程。协程是在一个线程栈空间内，通过栈寄存器esp和ebp(理解C语言的函数调用过程参考--http://blog.chinaunix.net/uid-23069658-id-3981406.html)，切换不同的代码片段，实现协程代码片段的并发。

　　协程避免了cpu切换线程上下文带来的开销，在io密集，cpu计算较少的情况下，优于线程。例如：web服务器就属于io密集型，比较适用于协程。

　　greenlet源码分析：

　　　　greenlet结构--greenlet.h

　　　

//greenlet对象最终对应的数据的C结构体，这里可以理解为python对象的属性  
typedef struct _greenlet {  
    PyObject_HEAD  
    char* stack_start;   //栈的顶部  将这里弄成null，标示已经结束了  
    char* stack_stop;    //栈的底部  
    char* stack_copy;     //栈保存到的内存地址  
    intptr_t stack_saved;   //栈保存在外面的大小  
    struct _greenlet* stack_prev;  //栈之间的上下层关系  
    struct _greenlet* parent;    //父对象  
    PyObject* run_info;   //其实也就是run对象  
    struct _frame* top_frame;   //这里可以理解为主要是控制python程序计数器  
    int recursion_depth;   //栈深度  
    PyObject* weakreflist;  
    PyObject* exc_type;  
    PyObject* exc_value;  
    PyObject* exc_traceback;  
    PyObject* dict;  
} PyGreenlet;

　　　　当在python中执行环境中创建greenlet对象的时候(gr = greenlet(func, parent_greenlet))，将会先调用green_new方法--greenlet.c

static PyObject* green_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
    PyObject* o;
    if (!STATE_OK)
        return NULL;
    
    o = type->tp_alloc(type, 0);
    if (o != NULL) {
        Py_INCREF(ts_current);
        ((PyGreenlet*) o)->parent = ts_current;//这里默认将parent设置为ts_current，也就是在哪一个greenlet环境里面创建的，那么新创建的greenlet的parent就是所属的环境greenlet
    }
    return o;
}

　　　　接着调用green_init方法

static int green_init(PyGreenlet *self, PyObject *args, PyObject *kwargs)
{
    PyObject *run = NULL;
    PyObject* nparent = NULL;
    static char *kwlist[] = {"run", "parent", 0};
    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OO:green", kwlist,
                     &run, &nparent))//从参数里解析run和parent

return -1;

    if (run != NULL) {
        if (green_setrun(self, run, NULL))
            return -1;
    }
    if (nparent != NULL && nparent != Py_None)
        return green_setparent(self, nparent, NULL);
    return 0;
}

static int green_setrun(PyGreenlet* self, PyObject* nrun, void* c)
{
    PyObject* o;
    if (PyGreenlet_STARTED(self)) {
        PyErr_SetString(PyExc_AttributeError,
                "run cannot be set "
                "after the start of the greenlet");
        return -1;
    }
    o = self->run_info;
    self->run_info = nrun;
    Py_XINCREF(nrun);
    Py_XDECREF(o);
    return 0;
}

static int green_setparent(PyGreenlet* self, PyObject* nparent, void* c)
{
    PyGreenlet* p;
    PyObject* run_info = NULL;
    if (nparent == NULL) {
        PyErr_SetString(PyExc_AttributeError, "can't delete attribute");
        return -1;
    }
    if (!PyGreenlet_Check(nparent)) {
        PyErr_SetString(PyExc_TypeError, "parent must be a greenlet");
        return -1;
    }
    for (p=(PyGreenlet*) nparent; p; p=p->parent) {
        if (p == self) {
            PyErr_SetString(PyExc_ValueError, "cyclic parent chain");
            return -1;
        }
        run_info = PyGreenlet_ACTIVE(p) ? p->run_info : NULL;
    }
    if (run_info == NULL) {
        PyErr_SetString(PyExc_ValueError, "parent must not be garbage collected");
        return -1;
    }
    if (PyGreenlet_STARTED(self) && self->run_info != run_info) {
        PyErr_SetString(PyExc_ValueError, "parent cannot be on a different thread");
        return -1;
    }
    p = self->parent;
    self->parent = (PyGreenlet*) nparent;
    Py_INCREF(nparent);
    Py_XDECREF(p);
    return 0;
}

接下来来看看greenlet对象对应的一些方法的定义：

 

/** End C API ****************************************************************/

static PyMethodDef green_methods[] = {
    {"switch", (PyCFunction)green_switch,
     METH_VARARGS | METH_KEYWORDS, green_switch_doc},
    {"throw",  (PyCFunction)green_throw,  METH_VARARGS, green_throw_doc},
    {"__getstate__", (PyCFunction)green_getstate, METH_NOARGS, NULL},
    {NULL,     NULL}        /* sentinel */
};

 

这里就可以看到最为重要的switch方法其实对应的green_switch方法，。。嗯，那么接下来就来对照着如下python代码来分析greenlet吧：

from greenlet import greenlet  
  
def test1(name):  
    print name  
    gr2.switch()  
    print "over"  
  
def test2():  
    print "test-2"  
    return "test2-fjsfjs"  
  
      
gr1 = greenlet(test1)  
gr2 = greenlet(test2)  
value = gr1.switch("fjsfjs")  
print value  

首先，得要先来看看greenlet模块的初始化：

PyMODINIT_FUNC
initgreenlet(void)
#endif
{
    PyObject* m = NULL;
    char** p = NULL;
    PyObject *c_api_object;
    static void *_PyGreenlet_API[PyGreenlet_API_pointers];

    GREENLET_NOINLINE_INIT();

#if PY_MAJOR_VERSION >= 3
    m = PyModule_Create(&greenlet_module_def);
#else
    m = Py_InitModule("greenlet", GreenMethods); /*主要是{"getcurrent", (PyCFunction)mod_getcurrent, METH_NOARGS, /*XXX*/ NULL}
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　* static PyObject* mod_getcurrent(PyObject* self)
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　*　{
	　　　　　　　　　　　　　　　　　　　　　　　　 *　　if (!STATE_OK)
		　　　　　　　　　　　　　　　　　　　　*　　　　return NULL;
	　　　　　　　　　　　　　　　　　　　　　　　　 *　　Py_INCREF(ts_current);
	　　　　　　　　　　　　　　　　　　　　　　　　　*　 return (PyObject*) ts_current;
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　* }
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　*/
#endif
    if (m == NULL)
    {
        INITERROR;
    }

    if (PyModule_AddStringConstant(m, "__version__", GREENLET_VERSION) < 0)
    {
        INITERROR;
    }

#if PY_MAJOR_VERSION >= 3
    ts_curkey = PyUnicode_InternFromString("__greenlet_ts_curkey");
    ts_delkey = PyUnicode_InternFromString("__greenlet_ts_delkey");
#if GREENLET_USE_TRACING
    ts_tracekey = PyUnicode_InternFromString("__greenlet_ts_tracekey");
    ts_event_switch = PyUnicode_InternFromString("switch");
    ts_event_throw = PyUnicode_InternFromString("throw");
#endif
#else
    ts_curkey = PyString_InternFromString("__greenlet_ts_curkey");
    ts_delkey = PyString_InternFromString("__greenlet_ts_delkey");
#if GREENLET_USE_TRACING
    ts_tracekey = PyString_InternFromString("__greenlet_ts_tracekey");
    ts_event_switch = PyString_InternFromString("switch");
    ts_event_throw = PyString_InternFromString("throw");
#endif
#endif
    if (ts_curkey == NULL || ts_delkey == NULL)
    {
        INITERROR;
    }
    if (PyType_Ready(&PyGreenlet_Type) < 0)
    {
        INITERROR;
    }
    PyExc_GreenletError = PyErr_NewException("greenlet.error", NULL, NULL);
    if (PyExc_GreenletError == NULL)
    {
        INITERROR;
    }
#if PY_MAJOR_VERSION >= 3 || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION >= 5)
    PyExc_GreenletExit = PyErr_NewException("greenlet.GreenletExit",
                        PyExc_BaseException, NULL);
#else
    PyExc_GreenletExit = PyErr_NewException("greenlet.GreenletExit",
                        NULL, NULL);
#endif
    if (PyExc_GreenletExit == NULL)
    {
        INITERROR;
    }

    ts_current = green_create_main();//创建root greenlet对象
    if (ts_current == NULL)
    {
        INITERROR;
    }

    Py_INCREF(&PyGreenlet_Type);
    PyModule_AddObject(m, "greenlet", (PyObject*) &PyGreenlet_Type);
    Py_INCREF(PyExc_GreenletError);
    PyModule_AddObject(m, "error", PyExc_GreenletError);
    Py_INCREF(PyExc_GreenletExit);
    PyModule_AddObject(m, "GreenletExit", PyExc_GreenletExit);
    PyModule_AddObject(m, "GREENLET_USE_GC", PyBool_FromLong(GREENLET_USE_GC));
    PyModule_AddObject(m, "GREENLET_USE_TRACING", PyBool_FromLong(GREENLET_USE_TRACING));

        /* also publish module-level data as attributes of the greentype. */
    for (p=copy_on_greentype; *p; p++) {
        PyObject* o = PyObject_GetAttrString(m, *p);
        if (!o) continue;
        PyDict_SetItemString(PyGreenlet_Type.tp_dict, *p, o);
        Py_DECREF(o);
    }

    /*
     * Expose C API
     */

    /* types */
    _PyGreenlet_API[PyGreenlet_Type_NUM] = (void *) &PyGreenlet_Type;

    /* exceptions */
    _PyGreenlet_API[PyExc_GreenletError_NUM] = (void *) PyExc_GreenletError;
    _PyGreenlet_API[PyExc_GreenletExit_NUM] = (void *) PyExc_GreenletExit;

    /* methods */
    _PyGreenlet_API[PyGreenlet_New_NUM] = (void *) PyGreenlet_New;
    _PyGreenlet_API[PyGreenlet_GetCurrent_NUM] =
        (void *) PyGreenlet_GetCurrent;
    _PyGreenlet_API[PyGreenlet_Throw_NUM] = (void *) PyGreenlet_Throw;
    _PyGreenlet_API[PyGreenlet_Switch_NUM] = (void *) PyGreenlet_Switch;
    _PyGreenlet_API[PyGreenlet_SetParent_NUM] =
        (void *) PyGreenlet_SetParent;

#ifdef GREENLET_USE_PYCAPSULE
    c_api_object = PyCapsule_New((void *) _PyGreenlet_API, "greenlet._C_API", NULL);
#else
    c_api_object = PyCObject_FromVoidPtr((void *) _PyGreenlet_API, NULL);
#endif
    if (c_api_object != NULL)
    {
        PyModule_AddObject(m, "_C_API", c_api_object);
    }

#if PY_MAJOR_VERSION >= 3
    return m;
#endif
}

static PyGreenlet* green_create_main(void)
{
    PyGreenlet* gmain;
    PyObject* dict = PyThreadState_GetDict();
    if (dict == NULL) {
        if (!PyErr_Occurred())
            PyErr_NoMemory();
        return NULL;
    }

    /* create the main greenlet for this thread */
    gmain = (PyGreenlet*) PyType_GenericAlloc(&PyGreenlet_Type, 0);
    if (gmain == NULL)
        return NULL;
    gmain->stack_start = (char*) 1;
    gmain->stack_stop = (char*) -1;//因为无符号，所以-1其实是最大的，这里在栈的复制拷贝的时候，就可以保证这里是最原始的，因为栈是由高地址向低地址扩展的  
    gmain->run_info = dict;
    Py_INCREF(dict);
    return gmain;
}

好了，greenlet模块的初始化完成了，那么接下来开始来分析python代码：

gr1 = greenlet(test1) 

创建了一个greenlet对象，传入了一个run函数，也就是test1函数，根据前面看到的构造和初始化函数可以知道，当前gr1的parent将会指向最开始初始化建立的greenlet对象。。。

接着看python代码：

 

gr2 = greenlet(test2)  

 

这里创建了greenlet对象，名字叫gr2，运行函数是test2，嗯，同理，它的parent也是初始化建立的greenlet对象。。。

value = gr1.switch("fjsfjs")  

这里调用gr1的switch方法，那么接下来就开始进入C语言的范围了。。我们来看看它的实现吧：

static PyObject* green_switch(
    PyGreenlet* self,
    PyObject* args,
    PyObject* kwargs)
{
    Py_INCREF(args);
    Py_XINCREF(kwargs);
    return single_result(g_switch(self, args, kwargs));
}

static PyObject *
g_switch(PyGreenlet* target, PyObject* args, PyObject* kwargs)
{
    /* _consumes_ a reference to the args tuple and kwargs dict,
       and return a new tuple reference */
    int err;
    PyObject* run_info;

    /* check ts_current */
    if (!STATE_OK) {
        Py_XDECREF(args);
        Py_XDECREF(kwargs);
        return NULL;
    }
    run_info = green_statedict(target);//
    if (run_info == NULL || run_info != ts_current->run_info) {
        Py_XDECREF(args);
        Py_XDECREF(kwargs);
        PyErr_SetString(PyExc_GreenletError, run_info
                ? "cannot switch to a different thread"
                : "cannot switch to a garbage collected greenlet");
        return NULL;
    }

    ts_passaround_args = args;
    ts_passaround_kwargs = kwargs;

    /* find the real target by ignoring dead greenlets,
       and if necessary starting a greenlet. */
    while (target) {
        if (PyGreenlet_ACTIVE(target)) {
            ts_target = target;
            err = g_switchstack();
            break;
        }
        if (!PyGreenlet_STARTED(target)) {
            void* dummymarker;
            ts_target = target;
            err = g_initialstub(&dummymarker);
            if (err == 1) {
                continue; /* retry the switch */
            }
            break;
        }
        target = target->parent;
    }

    /* For a very short time, immediately after the 'atomic'
       g_switchstack() call, global variables are in a known state.
       We need to save everything we need, before it is destroyed
       by calls into arbitrary Python code. */
    args = ts_passaround_args;
    ts_passaround_args = NULL;
    kwargs = ts_passaround_kwargs;
    ts_passaround_kwargs = NULL;
    if (err < 0) {
        /* Turn switch errors into switch throws */
        assert(ts_origin == NULL);
        Py_CLEAR(kwargs);
        Py_CLEAR(args);
    } else {
        PyGreenlet *origin;
#if GREENLET_USE_TRACING
        PyGreenlet *current;
        PyObject *tracefunc;
#endif
        origin = ts_origin;
        ts_origin = NULL;
#if GREENLET_USE_TRACING
        current = ts_current;
        if ((tracefunc = PyDict_GetItem(current->run_info, ts_tracekey)) != NULL) {
            Py_INCREF(tracefunc);
            if (g_calltrace(tracefunc, args ? ts_event_switch : ts_event_throw, origin, current) < 0) {
                /* Turn trace errors into switch throws */
                Py_CLEAR(kwargs);
                Py_CLEAR(args);
            }
            Py_DECREF(tracefunc);
        }
#endif
        Py_DECREF(origin);
    }

    /* We need to figure out what values to pass to the target greenlet
       based on the arguments that have been passed to greenlet.switch(). If
       switch() was just passed an arg tuple, then we'll just return that.
       If only keyword arguments were passed, then we'll pass the keyword
       argument dict. Otherwise, we'll create a tuple of (args, kwargs) and
       return both. */
    if (kwargs == NULL)
    {
        return args;
    }
    else if (PyDict_Size(kwargs) == 0)
    {
        Py_DECREF(kwargs);
        return args;
    }
    else if (PySequence_Length(args) == 0)
    {
        Py_DECREF(args);
        return kwargs;
    }
    else
    {
        PyObject *tuple = PyTuple_New(2);
        if (tuple == NULL) {
            Py_DECREF(args);
            Py_DECREF(kwargs);
            return NULL;
        }
        PyTuple_SET_ITEM(tuple, 0, args);
        PyTuple_SET_ITEM(tuple, 1, kwargs);
        return tuple;
    }
}

 

 

二 eventlet

　　1 eventlet

　　2 hub

　　3 greenthread

 

三 IO多路复用