1 #coding="utf-8"
2
3 import urllib2
4 import re
5 import threading
6 import time
7
8 """
9 抓取代理发布页的ip和port10 http://www.xici.net.co/nn/%d
11 """
12
13 proxylist = []
14
15
16 def get_proxy_from_cnproxy():
17 global proxylist
18
19 p = re.compile(r'<td><img alt="(.+?)" src=".+?" /></td>[\s\S]*?<td>(.+?)</td>[\s\S]*?<td>(.+?)</td>[\s\S]*?<td>[\s\S]*?<a href=".+?">.+?</a>[\s\S]*?</td>[\s\S]*?<td>.+?</td>[\s\S]*?<td>(.+?)</td>')
20
21 for i in range(1,2):
22 target = r"http://www.xici.net.co/nn/%d" %i
23 print target
24 req = urllib2.urlopen(target)
25 result = req.read()
26 matchs = p.findall(result)
27 for record in matchs:
28 addr = record[0]
29 ip = record[1]
30 port = record[2]
31 protocol = record[3]
32 l = [ip, port, protocol, addr]
33 #print l
34 proxylist.append(l)
35 print proxylist
36
37
38 class ProxyCheck(threading.Thread):
39 def __init__(self, proxylist, fname):
40 threading.Thread.__init__(self)
41 self.proxylist = proxylist
42 self.timeout = 5
43 self.test_url = "http://www.baidu.com/"
44 self.test_str = "030173"
45 self.checkedPProxyList = []
46 self.fname = fname
47
48 def checkProxy(self):
49 cookies = urllib2.HTTPCookieProcessor()
50 for proxy in self.proxylist:
51 proxy_handler = urllib2.ProxyHandler({"http":r'http://%s:%s'%(proxy[0],proxy[1])})
52 opener = urllib2.build_opener(cookies, proxy_handler)
53 opener.addheaders = [('user-agent', 'mozilla/5.0(iphone; u; cpu like mac os x; en) applewebkit/420+ (khtml, like gecko) version/3.0 mobile/1A537a safari/419.3')]
54 urllib2.install_opener(opener)
55 t1 = time.time()
56 try:
57 req = urllib2.urlopen(self.test_url, timeout = self.timeout)
58 result = req.read()
59 timeused = time.time() - t1
60 pos = result.find(self.test_str)
61 if pos > 1:
62 self.checkedPProxyList.append([proxy[0],proxy[1],proxy[2],proxy[3],timeused])
63 else:
64 continue;
65 except Exception,e:
66 print e.message
67 continue;
68
69 def sort(self):
70 sorted(self.checkedPProxyList,cmp=lambda x,y:cmp(x[4],y[4]))
71
72 def save(self):
73 f = open(self.fname, 'w+')
74 for proxy in self.checkedPProxyList:
75 f.write("%s:%s\t%s\t%s\t%s\n"%(proxy[0],proxy[1],proxy[2],proxy[3],str(proxy[4])))
76 f.close()
77
78 def run(self):
79 self.checkProxy()
80 self.sort()
81 self.save()
82
83
84 if __name__ == "__main__":
85 get_proxy_from_cnproxy()
86 t1 = ProxyCheck(proxylist,"test.txt")
87 t1.start()
