pyCurl example

#!/usr/bin/python
import pycurl,StringIO
def crawl(name):
    crl = pycurl.Curl()
    crl.setopt(pycurl.VERBOSE,0)     # No verbose info print
    crl.setopt(pycurl.NOSIGNAL, 1)   # avoid memory dump error
    crl.setopt(pycurl.FOLLOWLOCATION, 1)
    crl.setopt(pycurl.MAXREDIRS, 5)
    crl.fp = StringIO.StringIO()
    crl.setopt(pycurl.URL, url)
    crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
    crl.perform()
    page = crl.fp.getvalue()