抓取天涯文章的蜘蛛代码，刚经过更新（因为天涯页面HTML代码变化）

#_*_coding:utf-8-*-
import urllib2
import traceback
import codecs
from BeautifulSoup import BeautifulSoup

def openSoup(url,code):
    page = urllib2.urlopen(url)
    soup = BeautifulSoup(page,fromEncoding=code)#,fromEncoding="gb2312"
    #soup = BeautifulSoup(page,code)
    return soup

def getContentFromDiv(contents):
    s = ""
    for content in contents:
        try:
            s += content
        except:
            pass
    
    s = s.lstrip().rstrip()
    if len(s) < 50:
        return ""
    else:
        return "    "+s+"\r\n"+"\r\n"

def readHtml(soup,fp,authname):
    pageContent = ""
    item = soup.find(name='div', attrs={'class':'bbs-content clearfix'})
    if item != None:
        pageContent += getContentFromDiv(item.contents)

    items = soup.findAll(name='div', attrs={'class':'atl-item'})
    for item in items:
        userItem = item.find(name='a', attrs={'class':'js-vip-check'})
        if userItem == None or userItem.contents[0] != authname:
            continue

        contentItem = item.find(name='div', attrs={'class':'bbs-content'})
        pageContent += getContentFromDiv(contentItem.contents)
    
    fp.write(pageContent)
   
def getNextPage(soup,pno):
    nextlink = soup.find(name="a",attrs={"class":"js-keyboard-next"})
    if nextlink != None:
        return "http://bbs.tianya.cn"+nextlink["href"]
    else:
        return 'OVER'

def getHtml(url,filename,authname):
    p = 1
    fp = codecs.open(filename,'w','utf-8')
    while True:
        soup = openSoup(url,'utf-8')
        readHtml(soup,fp,authname)
        url = getNextPage(soup,p+1)
        if url == 'OVER' :
            break
        print 'PAGE '+str(p)+' OK'
        p = p + 1
       
    print 'It\'s Over'
    fp.close()


if __name__ == '__main__':
    getHtml('http://bbs.tianya.cn/post-no05-143258-1.shtml','krzc.txt',u'关河五十州')
    #getHtml('http://bbs.tianya.cn/post-no05-143258-1036.shtml','krzc.txt',u'关河五十州')