freebuf站信息抓取实例

# coding: utf-8

import urllib2
from bs4 import BeautifulSoup


class MySpider(object):

    def getHtml(self,url):
        '''
        获取页面源代码
        '''
        req=urllib2.Request(url)
        req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36')
        res=urllib2.urlopen(req).read()
        return res

    def getPage(self,url,total_page):
        '''
        生成多个链接
        '''
        all_page=[]
        for i in range(1,total_page+1):
            link=url+'page/%s' % i
            all_page.append(link)
        return all_page

    def get_tdiv(self,res):
        '''
        获取加载下来页面的每个tbody标签
        '''
        soup=BeautifulSoup(res)
        tdiv=soup.find_all('div',class_='news_inner news-list')
        return tdiv
    
    def Info(self,tdiv):
        '''
        把需要的信息写入到字典中去
        '''
        info={}
        info['title']=tdiv.a.img.attrs['title']
        res=tdiv.find_all('dd',class_='text')
        info['desc']=res[0].string.strip('\n+').lstrip(' ')
        info['picUrl']=tdiv.a.img.attrs['src']
        info['url']=tdiv.dt.a.attrs['href']
        return info


def main():
    minfo=[]
    url='http://www.freebuf.com/'
    ms=MySpider()
    all_links=ms.getPage(url,2)
    try:
        for link in all_links:
            print r'正在加载页面:'+link
            res=ms.getHtml(link)
            tdiv=ms.get_tdiv(res)
            for td in tdiv:
                nn=ms.Info(td)
                print nn['title']
                print nn['desc']
                print nn['picUrl']
                print nn['url']
                print '----------------------------'
    except Exception,e:
        print str(e)


if __name__=='__main__':
    main()