Python 爬取SeeBug poc 

 1 #!/usr/bin/env python
 2 # -*- coding: utf-8 -*-
 3 # @Date    : 2017-08-24 21:42:52
 4 # @Author  : EnderZhou (zptxwd@gmail.com)
 5 # @Link    : http://www.cnblogs.com/enderzhou/
 6 # @Version : $Id$
 7 
 8 import requests
 9 from bs4 import BeautifulSoup as bs
10 import threading
11 import Queue
12 import urllib
13 # import os
14 import time
15 
16 main_url ='https://www.seebug.org/vuldb/vulnerabilities?category=&order_time=1&order_rank=1&has_all=default&has_vm=default&submitTime=all&has_affect=default&has_poc=true&has_detail=default&level=all&page='#+pagenumber
17 
18 bug_url = 'https://www.seebug.org/vuldb/ssvid-'#96358
19 
20 dl_url = 'https://www.seebug.org/vuldb/downloadPoc/'#96358
21 
22 
23 #SeeBug防爬机制教强，若仅修改cookie无效，请替换全部header头信息
24 headers = {
25 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
26 'Accept-Encoding':'gzip, deflate, br',
27 'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
28 'AlexaToolbar-ALX_NS_PH':'AlexaToolbar/alx-4.0.1',
29 'Cache-Control':'max-age=0',
30 'Connection':'keep-alive',
31 'Cookie':'请填写自己的cookie',
32 'Referer':'https://www.seebug.org/vuldb/vulnerabilities',
33 'Upgrade-Insecure-Requests':'1',
34 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36',
35 }
36 
37 class SeeBugPoc(threading.Thread):
38     def __init__(self,queue):
39         threading.Thread.__init__(self)
40         self._queue = queue
41 
42     def run(self):
43         while not self._queue.empty():
44             url = self._queue.get_nowait()
45             self.spidet(url)
46 
47     def spidet(self,url):
48         r = requests.get(url=url,headers=headers)
49         soup = bs(r.content,'html.parser')
50         vullist = soup.find_all(name='a',attrs={'class':'vul-title'})
51         for u in vullist:
52             name = u['href'].split('/')[-1]
53             l = dl_url+u['href'].split('-')[-1]
54             print name + '----' + l
55             d = requests.get(url=l,headers=headers)
56             f =  open(name+'.txt','w') 
57             f.write(d.content)
58             f.close
59             time.sleep(7.5)
60 #SeeBug防爬机制教强，此处设置延时7.5秒。后续看心情是否更新添加代理功能
61 
62 def main():
63     queue = Queue.Queue()
64     for i in range(1,2185):
65         queue.put(main_url+str(i))
66 
67     threads = []
68     thread_count = 1
69 
70     for i in range(thread_count):
71         threads.append(SeeBugPoc(queue))
72 
73     for t in threads:
74         t.start()
75 
76     for t in threads:
77         t.join()
78 
79 if __name__ == '__main__':
80     main()

 

posted on 2017-08-28 17:11  武诚治  阅读(649)  评论(0)    收藏  举报

导航