109-python-某视频网站爬取数据

import re
from urllib.request import urlopen,Request
from multiprocessing import Pool

def get_page(url,pattern):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
    req = Request(url=url, headers=headers)
    response = urlopen(req).read().decode('utf-8')
    return pattern,response   # 正则表达式编译结果 网页源代码

def parse_page(info):
    pattern,page_content=info
    res=re.findall(pattern,page_content)
    for item in res:
        dic={
            'index':item[0].strip(),
            'title':item[1].strip(),
            'actor':item[2].strip(),
            'time':item[3].strip(),
            'score': '%s%s'%(item[4].strip(), item[5].strip())
        }
        print(dic)
if __name__ == '__main__':
    # regex = r'<dd>.*?<.*?class="board-index.*?>(\d+)</i>.*?title="(.*?)".*?class="movie-item-info".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>'
    # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>'
    # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p><p class="score">.*?<i class="integer">(\d+\.?)</i><i class="fraction">(\d+)</i></p>.*?'
    # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title=".*?".*">.*?(.*?)</a></p>.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>'
    # regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>'
    regex = r'<dd>.*?<i class="board-index.*?>(\d+)</i>.*?<p class="name">.*?<a.*?title="(.*?)".*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>.*?<p class="score"><i class="integer">(.*?)</i>.*?<i class="fraction">(.*?)</i></p>.*?'

    pattern1=re.compile(regex,re.S)
    url_dic={'http://maoyan.com/board/7':pattern1}
    p=Pool()
    res_l=[]
    for url,pattern in url_dic.items():
        res=p.apply_async(get_page,args=(url,pattern),callback=parse_page)
        res_l.append(res)

    for i in res_l:
        i.get()