1 import requests
2 from requests.exceptions import RequestException
3 import re
4 import json
5 from multiprocessing import Pool
6
7 def get_one_page(url):
8 try:
9 headers={'user-agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0) 32-bit IE on 64-bit Windows 7'}
10 response=requests.get(url,headers=headers)
11 if response.status_code==200:
12 return response.text
13 return None
14 except RequestException:
15 return None
16
17 def parse_one_page(html):
18 pattern=re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>',re.S)
19 items=re.findall(pattern,html)
20 for item in items:
21 yield{
22 'index':item[0],
23 'image':item[1],
24 'title':item[2],
25 'actor':item[3].strip()[3:],
26 'time':item[4].strip()[5:],
27 'score':item[5]+item[6]
28 }
29
30 def write_to_file(content):
31 with open('result.txt','a',encoding='utf-8') as f:
32 f.write(json.dumps(content,ensure_ascii=False)+'\n')
33 f.close()
34
35
36 def main(offset):
37 url='http://maoyan.com/board/4?offset='+str(offset)
38 html=get_one_page(url)
39 for item in parse_one_page(html):
40 print(item)
41 write_to_file(item)
42
43 if __name__=='__main__':
44 pool=Pool()
45 pool.map(main,[i*10 for i in range(10)])
