lxml_time_代理

 

  1 import requests
  2 from pyquery import PyQuery as pq
  3 import json
  4 import jsonpath
  5 from lxml import etree
  6 import os
  7 import re
  8 import time
  9 
 10 html = '''
 11 <div>
 12     <ul>
 13          <li class="item-0">first item</li>
 14          <li class="item-1"><a href="link2.html">second item</a></li>
 15          <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
 16          <li class="item-1 active"><a href="link4.html">fourth item</a></li>
 17          <li class="item-0"><a href="link5.html">fifth item</a></li>
 18      </ul>
 19 </div>
 20 '''
 21 
 22 # html = requests.get('http://news.4399.com/gonglue/lscs/kptj/').content.decode('gbk')
 23 num = 0
 24 # def pq方法(url):
 25 #     global num
 26 #     html=requests.get(url).content.decode('gbk')
 27 #     doc = pq(html)
 28 #     items = doc('#dq_list > li').items()
 29 #     # print(doc)
 30 #     # print(type(doc))
 31 #     for item in items:
 32 #         url=item.find('img').attr('lz_src')
 33 #         num+=1
 34 #         print(str(num),url)
 35 #         url_content=requests.get(url).content
 36 #         name = item.find('.kp-name').text()
 37         
 38 #         with open('e:/py3/002/'+'{:0>4}'.format(str(num))+name+'.jpg','wb') as file:
 39 #             file.write(url_content)
 40 #         # print(url,name)
 41 
 42 def transformCodec(re_data):#ascii (gbk) 转 unicode
 43     try:
 44         re_data = re_data.decode('gbk')
 45     except Exception as error:
 46         print (error)
 47         print ('delete illegal string,try again...')
 48         
 49         pos = re.findall(r'decodebytesinposition([\d]+)-([\d]+):illegal',str(error).replace(' ',''))
 50         if len(pos)==1:
 51             re_data = re_data[0:int(pos[0][0])]+re_data[int(pos[0][1]):]
 52             re_data = transformCodec(re_data)
 53             return re_data
 54     return re_data
 55 
 56 
 57 def lxml方法(url):
 58     global num
 59     header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
 60     content=requests.get(url,headers=header).content
 61     html=content.decode('utf-8')
 62     # print(html.status_code)
 63     # print(content)
 64     # print(html)    
 65     r=etree.HTML(html)
 66     # items=r.xpath("//div[@class='box10-content']//ul[@id='dq_list']/li/a/img/@lz_src")
 67     items=r.xpath("//div[@id='list']/table//tr")
 68     # print(items)
 69     for item in items:
 70         dl_ip=item.xpath("./td[1]/text()")
 71         dl_port=item.xpath("./td[2]/text()")
 72         dl_name=item.xpath("./td[5]/text()")
 73         num+=1
 74         dl_ip=dl_ip[0]+":" if len(dl_ip)>=1 else ''
 75         dl_port=dl_port[0]+"#" if len(dl_port)>=1 else ''
 76         dl_name=dl_name[0] if len(dl_name)>=1 else ''
 77         
 78         # print(len(dl_ip))
 79         # print(dl_ip)
 80         # print(r'{}{}{}'.format(dl_ip,dl_port,dl_name))
 81         with open("proxy.txt",'a',encoding='utf-8') as file:
 82             file.write('{}{}{}\n'.format(dl_ip,dl_port,dl_name))
 83         # lzcontent=requests.get(lzsrc).content
 84         # with open('e:/py3/004/'+'{:0>4}'.format(str(num))+'_'+kpname+'.jpg','wb')as file:
 85         #     file.write(lzcontent)
 86 
 87 
 88 
 89 
 90 
 91 if __name__ == '__main__':
 92     with open("proxy.txt", 'w', encoding='utf-8') as file:
 93             file.write(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2])+'_采集：\n')
 94     # url='https://www.kuaidaili.com/free/inha/1/'
 95     for i in range(1,11):
 96         print('第'+str(i)+'次：\n')
 97         url2 = r'https://www.kuaidaili.com/free/inha/'+str(i)+r'/'
 98         print(url2)
 99         lxml方法(url2)
100         time.sleep(5)
101 
102     # header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
103     # pq方法()
104     # print(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2]))
105     print(str(num)+' ok!')
106 
107 
108     # 创建目录
109     '''
110     for dirnum in range(1,100):
111         dirnum2='{:0>3}'.format(str(dirnum))
112         mkpath="e:\\py3\\{}\\".format(dirnum2)
113         print(mkpath)
114         print('已存在！') if os.path.exists(mkpath) else os.makedirs(mkpath)
115     '''

 

posted @ 2018-10-28 17:14  快乐多巴胺  阅读(181)  评论(0)    收藏  举报