新浪微博爬虫

　　这里给出新浪微博电脑端（http://weibo.com）和手机端（http://weibo.cn）的爬虫代码。

　　新浪微博电脑端和手机端的登陆机制并不一样，本文不做详细介绍，给只给出代码

 

新浪微博 手机端（http://weibo.cn）

#/usr/bin/env python
# coding=utf-8
import urllib2
import urllib
import cookielib
import time
import lxml.html as HTML
 
class Fetcher(object):
    def __init__(self, username=None, pwd=None, cookie_filename=None):
        self.cj = cookielib.LWPCookieJar()
        if cookie_filename is not None:
            self.cj.load(cookie_filename)
        self.cookie_processor = urllib2.HTTPCookieProcessor(self.cj)
        self.opener = urllib2.build_opener(self.cookie_processor, urllib2.HTTPHandler)
        urllib2.install_opener(self.opener)
         
        self.username = username
        self.pwd = pwd
        self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/14.0.1',
                        'Referer':'','Content-Type':'application/x-www-form-urlencoded'}
     
    def get_rand(self, url):
        headers = {'User-Agent':'Mozilla/5.0 (Windows;U;Windows NT 5.1;zh-CN;rv:1.9.2.9)Gecko/20100824 Firefox/3.6.9',
                   'Referer':''}
        req = urllib2.Request(url ,urllib.urlencode({}), headers)
        resp = urllib2.urlopen(req)
        login_page = resp.read()
        rand = HTML.fromstring(login_page).xpath("//form/@action")[0]
        passwd = HTML.fromstring(login_page).xpath("//input[@type='password']/@name")[0]
        vk = HTML.fromstring(login_page).xpath("//input[@name='vk']/@value")[0]
        return rand, passwd, vk
     
    def login(self, username=None, pwd=None, cookie_filename=None):
        if self.username is None or self.pwd is None:
            self.username = username
            self.pwd = pwd
        assert self.username is not None and self.pwd is not None
         
        url = 'http://3g.sina.com.cn/prog/wapsite/sso/login.php?ns=1&revalid=2&backURL=http%3A%2F%2Fweibo.cn%2F&backTitle=%D0%C2%C0%CB%CE%A2%B2%A9&vt='
        rand, passwd, vk = self.get_rand(url)
        data = urllib.urlencode({'mobile': self.username,
                                 passwd: self.pwd,
                                 'remember': 'on',
                                 'backURL': 'http://weibo.cn/',
                                 'backTitle': '新浪微博',
                                 'vk': vk,
                                 'submit': '登录',
                                 'encoding': 'utf-8'})
        url = 'http://3g.sina.com.cn/prog/wapsite/sso/' + rand
        req = urllib2.Request(url, data, self.headers)
        resp = urllib2.urlopen(req)
        page = resp.read()
        link = HTML.fromstring(page).xpath("//a/@href")[0]
        if not link.startswith('http://'): link = 'http://weibo.cn/%s' % link
        req = urllib2.Request(link, headers=self.headers)
        urllib2.urlopen(req)
        if cookie_filename is not None:
            self.cj.save(filename=cookie_filename)
        elif self.cj.filename is not None:
            self.cj.save()
        print 'login success!'
         
    def fetch(self, url, timeout):
        req = urllib2.Request(url, headers=self.headers)
        return urllib2.urlopen(req, None, timeout).read()
if __name__ == '__main__'
    username='xx@xx.com';
    password='xxxx;
    fetcher = Fetcher(username, password);
    fetcher.login();
    seed_url='http://weibo.cn/xiaomishouji?filter=0';
    try :
        htmlContent = fetcher.fetch(seed_url, 3):
    except Exception,e :
        print 'time out';

新浪微博电脑端

#/usr/bin/env python
# coding=utf-8
import re
import json
import urllib
import base64
import rsa
import binascii
import urllib2
import cookielib

def sServerData(serverData):
    "Search the server time & nonce from server data"

    p = re.compile('\((.*)\)')
    jsonData = p.search(serverData).group(1)
    data = json.loads(jsonData)
    serverTime = str(data['servertime'])
    nonce = data['nonce']
    pubkey = data['pubkey']#
    rsakv = data['rsakv']#
    print "Server time is:", serverTime
    print "Nonce is:", nonce
    return serverTime, nonce, pubkey, rsakv
 
def sRedirectData(text):
    p = re.compile('location\.replace\([\'"](.*?)[\'"]\)')
    loginUrl = p.search(text).group(1)
    print 'loginUrl:',loginUrl
    return loginUrl
def PostEncode(userName, passWord, serverTime, nonce, pubkey, rsakv):
    "Used to generate POST data"

    encodedUserName = GetUserName(userName)
    encodedPassWord = get_pwd(passWord, serverTime, nonce, pubkey)
    postPara = {
        'entry': 'weibo',
        'gateway': '1',
        'from': '',
        'savestate': '7',
        'userticket': '1',
        'ssosimplelogin': '1',
        'vsnf': '1',
        'vsnval': '',
        'su': encodedUserName,
        'service': 'miniblog',
        'servertime': serverTime,
        'nonce': nonce,
        'pwencode': 'rsa2',
        'sp': encodedPassWord,
        'encoding': 'UTF-8',
        'prelt': '115',
        'rsakv': rsakv,     
        'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
        'returntype': 'META'
    }
    postData = urllib.urlencode(postPara)
    return postData
def GetUserName(userName):
    "Used to encode user name"

    userNameTemp = urllib.quote(userName)
    userNameEncoded = base64.encodestring(userNameTemp)[:-1]
    return userNameEncoded
 
def get_pwd(password, servertime, nonce, pubkey):
    rsaPublickey = int(pubkey, 16)
    key = rsa.PublicKey(rsaPublickey, 65537) 
    message = str(servertime) + '\t' + str(nonce) + '\n' + str(password) 
    passwd = rsa.encrypt(message, key) 
    passwd = binascii.b2a_hex(passwd) 
    return passwd

class Fetcher:
    def __init__(self, user, pwd, enableProxy = False):
        print "Initializing WeiboLogin..."
        self.userName = user
        self.passWord = pwd
        self.enableProxy = enableProxy
        self.serverUrl = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.11)&_=1379834957683"
        self.loginUrl = "http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)"
        self.postHeader = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0'}
    def Login(self):
        self.EnableCookie(self.enableProxy)
        serverTime, nonce, pubkey, rsakv = self.GetServerTime()
        postData = PostEncode(self.userName, self.passWord, serverTime, nonce, pubkey, rsakv)#
        print "Post data length:\n", len(postData)  
        req = urllib2.Request(self.loginUrl, postData, self.postHeader)
        print "Posting request..."
        result = urllib2.urlopen(req)
        text = result.read()
        try:
            loginUrl = sRedirectData(text)
            urllib2.urlopen(loginUrl)
        except:
            print 'Login error!'
            return False
        print 'Login sucess!'
        return True
    def EnableCookie(self, enableProxy):
        "Enable cookie & proxy (if needed)."
        cookiejar = cookielib.LWPCookieJar()
        cookie_support = urllib2.HTTPCookieProcessor(cookiejar)  
        if enableProxy:
            proxy_support = urllib2.ProxyHandler({'http':'http://xxxxx.pac'})
            opener = urllib2.build_opener(proxy_support, cookie_support, urllib2.HTTPHandler)
            print "Proxy enabled"
        else:
            opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler) 
        urllib2.install_opener(opener)
    def GetServerTime(self):
        "Get server time and nonce, which are used to encode the password"

        print "Getting server time and nonce..."
        serverData = urllib2.urlopen(self.serverUrl).read()
        print serverData  
        try:
            serverTime, nonce, pubkey, rsakv = sServerData(serverData)
            return serverTime, nonce, pubkey, rsakv
        except:
            print 'Get server time & nonce error!'
            return None
    def fetch(self, url, timeout) :
        req = urllib2.Request(url, headers=self.postHeader)
        return urllib2.urlopen(req, None, timeout).read()

if __name__ == '__main__':
    username = 'xx@xx.com'
    passwd = 'xxxx'
    fetcher = Fetcher(username, passwd)
    if fetcher.Login() == True:
        print "Login success!"
    seed_url='http://weibo.com/p/1006061771925961/weibo'
    try :
        htmlContent = fetcher.fetch(seed_url, 3);
    except Exception,e :
        print 'time out';