2019.6.18

昨日作业

自动登录抽屉网

 1 from selenium import webdriver
 2 import time
 3 
 4 driver = webdriver.Chrome()
 5 driver.maximize_window()
 6 
 7 try:
 8     driver.get('https://dig.chouti.com/')
 9     driver.implicitly_wait(10)
10     time.sleep()
11 
12     login_bin = driver.find_element_by_id('login_btn')
13     login_bin.click()
14     time.sleep(2)
15 
16     phone = driver.find_element_by_class_name('login-phone')
17     phone.send_keys('17718268511')
18 
19     pwd = driver.find_element_by_class_name('pwd-password-imput')
20     pwd.send_keys('tututu')
21 
22     login_submit = driver.find_element_by_class_name('btn-large')
23     login_submit.click()
24     time.sleep(20)
25 
26 #捕获异常并打印
27 except Exception as e:
28     print(e)
29 
30 finally:
31     driver.close()
View Code

 

今日内容

1、selenium剩余用法

2、selenium万能登陆破解

3、破解极验滑动验证码

 

1、selenium剩余用法

selenuim选择器之Xpath

  1 from selenium import webdriver
  2 
  3 driver = webdriver.Chrome()
  4 
  5 try:
  6     driver.get('https://doc.scrapy.org/en/latest/_static/selectors-sample1.html')
  7     driver.implicitly_wait(5)
  8 
  9     根据Xpath语法查找元素
 10     /从根节点开始找第一个
 11     html = driver.find_element_by_xpath('/html')
 12     print(html.tag_name)
 13 
 14     #//从根节点找任意一个节点
 15     div = driver.find_element_by_xpath('//div')
 16     print(div.tag_name)
 17 
 18 
 19     #@
 20     #查找id为images的div节点
 21     div = driver.find_element_by_xpath('//div[@id="images"]')
 22     print(div.tag_name)
 23     print(div.text)
 24 
 25     #找到一个a节点
 26     a = driver.find_element_by_xpath('//a')
 27     print(a.tag_name)
 28 
 29     #找到所有a节点
 30     a_s = driver.find_elements_by_xpath('//a')
 31     print(a_s)
 32 
 33     #找到第一个a节点的href属性
 34     a = driver.find_element_by_xpath('//a').get_attribute('href')
 35     print(a)
 36 
 37 finally:
 38     driver.close()
 39 
 40 
 41     找到所有a节点
 42 
 43 
 44 '''
 45 
 46 添加cookie
 47 '''
 48 from selenium import webdriver
 49 driver = webdriver.Chrome()
 50 import time
 51 
 52 
 53 
 54 try:
 55     driver.implicitly_wait(10)
 56     driver.get('https://www.zhihu.com/explore')
 57     print(driver.get_cookie())
 58 
 59     time.sleep(10)
 60 
 61 
 62 finally:
 63     driver.close()
 64 
 65 
 66 
 67 '''
 68 #
 69 # 选项卡
 70 #
 71 # '''
 72 
 73 import time
 74 from selenium import webdriver
 75 
 76 browser = webdriver.Chrome()
 77 
 78 try:
 79     browser.get('https://')
 80 
 81 
 82 
 83 finally:
 84     browser.close()
 85 
 86 
 87 
 88 '''
 89 前进、后退
 90 '''
 91 from selenium import webdriver
 92 import time
 93 
 94 driver = webdriver.Chrome()
 95 
 96 
 97 try:
 98     driver.implicitly_wait(10)
 99     driver.get('https://www.jd.com/')
100     driver.get('https://www.baidu.com/')
101     driver.get('https://www.cnblogs.com/')
102 
103     time.sleep(2)
104 
105     # 回退操作
106     driver.back()
107     time.sleep(1)
108     # 前进操作
109     driver.forward()
110     time.sleep(1)
111     driver.back()
112     time.sleep(10)
113 
114 finally:
115     driver.close()

 

selenium剩余操作

  1 ''''''
  2 '''
  3 点击、清除操作
  4 '''
  5 # from selenium import webdriver
  6 # from selenium.webdriver.common.keys import Keys
  7 # import time
  8 #
  9 # driver = webdriver.Chrome(r'D:\BaiduNetdiskDownload\chromedriver_win32\chromedriver.exe')
 10 #
 11 # try:
 12 #     driver.implicitly_wait(10)
 13 #     # 1、往jd发送请求
 14 #     driver.get('https://www.jd.com/')
 15 #     # 找到输入框输入围城
 16 #     input_tag = driver.find_element_by_id('key')
 17 #     input_tag.send_keys('围城')
 18 #     # 键盘回车
 19 #     input_tag.send_keys(Keys.ENTER)
 20 #     time.sleep(2)
 21 #     # 找到输入框输入墨菲定律
 22 #     input_tag = driver.find_element_by_id('key')
 23 #     input_tag.clear()
 24 #     input_tag.send_keys('墨菲定律')
 25 #     # 找到搜索按钮点击搜索
 26 #     button = driver.find_element_by_class_name('button')
 27 #     button.click()
 28 #     time.sleep(10)
 29 #
 30 # finally:
 31 #     driver.close()
 32 
 33 
 34 '''
 35 获取cookies  （了解）
 36 '''
 37 # from selenium import webdriver
 38 # import time
 39 #
 40 # driver = webdriver.Chrome(r'D:\BaiduNetdiskDownload\chromedriver_win32\chromedriver.exe')
 41 #
 42 # try:
 43 #     driver.implicitly_wait(10)
 44 #     driver.get('https://www.zhihu.com/explore')
 45 #     print(driver.get_cookies())
 46 #
 47 #     time.sleep(10)
 48 # finally:
 49 #     driver.close()
 50 
 51 '''
 52 选项卡
 53 '''
 54 #选项卡管理：切换选项卡，有js的方式windows.open,有windows快捷键：
 55 # ctrl+t等，最通用的就是js的方式
 56 # import time
 57 # from selenium import webdriver
 58 #
 59 # browser = webdriver.Chrome()
 60 # try:
 61 #     browser.get('https://www.baidu.com')
 62 #
 63 #     # execute_script: 执行javascrpit代码
 64 #     # 弹窗操作
 65 #     # browser.execute_script('alert("tank")')
 66 #     # 新建浏览器窗口
 67 #     browser.execute_script(
 68 #         '''
 69 #         window.open();
 70 #         '''
 71 #     )
 72 #     time.sleep(1)
 73 #     print(browser.window_handles)  # 获取所有的选项卡
 74 #     # 切换到第二个窗口
 75 #     # 新:
 76 #     browser.switch_to.window(browser.window_handles[1])
 77 #     # 旧:
 78 #     # browser.switch_to_window(browser.window_handles[1])
 79 #
 80 #     # 第二个窗口往淘宝发送请求
 81 #     browser.get('https://www.taobao.com')
 82 #     time.sleep(5)
 83 #
 84 #     # 切换到第一个窗口
 85 #     browser.switch_to_window(browser.window_handles[0])
 86 #     browser.get('https://www.sina.com.cn')
 87 #
 88 #     time.sleep(10)
 89 # finally:
 90 #     browser.close()
 91 
 92 
 93 '''
 94 ActionChangs动作链
 95 '''
 96 # from selenium import webdriver
 97 # from selenium.webdriver import ActionChains
 98 # import time
 99 #
100 # driver = webdriver.Chrome()
101 # driver.implicitly_wait(10)
102 # driver.get('http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')
103 #
104 # try:
105 #
106 #     # driver.switch_to_frame('iframeResult')
107 #     # 切换到id为iframeResult的窗口内
108 #     driver.switch_to.frame('iframeResult')
109 #
110 #     # 源位置
111 #     draggable = driver.find_element_by_id('draggable')
112 #
113 #     # 目标位置
114 #     droppable = driver.find_element_by_id('droppable')
115 #
116 #     # 调用ActionChains，必须把驱动对象传进去
117 #     # 得到一个动作链对象，复制给一个变量
118 #     actions = ActionChains(driver)
119 #
120 #     # 方式一: 机器人
121 #     # 瞬间把源图片位置秒移到目标图片位置
122 #     # actions.drag_and_drop(draggable, droppable)  # 编写一个行为
123 #     # actions.perform()  # 执行编写好的行为
124 #
125 #
126 #     # 方式二: 模拟人的行为
127 #     source = draggable.location['x']
128 #     target = droppable.location['x']
129 #     print(source, target)
130 #
131 #     distance = target - source
132 #     print(distance)
133 #
134 #     # perform：每个动作都要调用perform执行
135 #
136 #     # 点击并摁住源图片
137 #     ActionChains(driver).click_and_hold(draggable).perform()
138 #
139 #     s = 0
140 #     while s < distance:
141 #         # 执行位移操作
142 #         ActionChains(driver).move_by_offset(xoffset=2, yoffset=0).perform()
143 #         s += 2
144 #
145 #     # 释放动作链
146 #     ActionChains(driver).release().perform()
147 #
148 #     time.sleep(10)
149 #
150 #
151 # finally:
152 #     driver.close()
153 
154 
155 '''
156 前进、后退
157 '''
158 # from selenium import webdriver
159 # import time
160 #
161 # driver = webdriver.Chrome()
162 #
163 # try:
164 #     driver.implicitly_wait(10)
165 #     driver.get('https://www.jd.com/')
166 #     driver.get('https://www.baidu.com/')
167 #     driver.get('https://www.cnblogs.com/')
168 #
169 #     time.sleep(2)
170 #
171 #     # 回退操作
172 #     driver.back()
173 #     time.sleep(1)
174 #     # 前进操作
175 #     driver.forward()
176 #     time.sleep(1)
177 #     driver.back()
178 #     time.sleep(10)
179 #
180 # finally:
181 #     driver.close()

 

2、selenium万能登陆破解

 1 from selenium import webdriver
 2 from selenium.webdriver import ChromeOptions
 3 import time
 4 r'''
 5 步骤:
 6     1、打开文件的查看，显示隐藏文件
 7     2、找到C:\Users\administortra\AppData\Local\Google\Chrome\User Data
 8         删除Default文件
 9     3、重新打开浏览器，并登陆百度账号
10         - 此时会创建一个新的Default缓存文件
11     4、添加cookies
12     5、关闭谷歌浏览器后执行程序
13 '''
14 # 获取options对象，参数对象
15 options = ChromeOptions()
16 
17 # 获取cookies保存路径
18 # 'C:\Users\administortra\AppData\Local\Google\Chrome\User Data'
19 profile_directory = r'--user-data-dir=C:\Users\administortra\AppData\Local\Google\Chrome\User Data'
20 
21 # 添加用户信息目录
22 options.add_argument(profile_directory)
23 
24 # 把参数加载到当前驱动中  chrome_options默认参数，用来接收options对象
25 driver = webdriver.Chrome(chrome_options=options)
26 
27 try:
28     driver.implicitly_wait(10)
29     driver.get('https://www.baidu.com/')
30     '''
31     BDUSS:*****
32     '''
33     # 添加用户cookies信息
34     # name、value必须小写
35     driver.add_cookie({"name": "BDUSS", "value": "用户session字符串"})
36 
37     # 刷新操作
38     driver.refresh()
39 
40     time.sleep(10)
41 
42 finally:
43     driver.close()

 

今日作业

1、总结课堂知识点，写博客

2、爬取京东商品信息

3、滑动验证（提高题）

 

爬取京东商品信息

  1 from selenium import webdriver
  2 from selenium.webdriver.support.ui import WebDriverWait
  3 from selenium.webdriver.support import expected_conditions as EC
  4 from selenium.webdriver.common.by import By
  5 from selenium.webdriver.common.keys import Keys
  6 import time
  7 import json
  8 import csv
  9 import random
 10 
 11 
 12 # 声明一个谷歌驱动器，并设置不加载图片，间接加快访问速度
 13 options = webdriver.ChromeOptions()
 14 options.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2})
 15 browser = webdriver.Chrome(options=options)
 16 # url
 17 url = 'https://www.jd.com/'
 18 # 声明一个list，存储dict
 19 data_list = []
 20 
 21 
 22 def start_spider():
 23     # 请求url
 24     browser.get(url)
 25     # 获取输入框的id，并输入关键字python爬虫
 26     browser.find_element_by_id('key').send_keys('python爬虫')
 27     # 输入回车进行搜索
 28     browser.find_element_by_id('key').send_keys(Keys.ENTER)
 29     # 显示等待下一页的元素加载完成
 30     WebDriverWait(browser, 1000).until(
 31         EC.presence_of_all_elements_located(
 32             (By.CLASS_NAME, 'pn-next')
 33         )
 34     )
 35     # 先获取一个有多少页
 36     all_page = eval(browser.find_element_by_css_selector('span.p-skip em b').text)
 37     # 设置一个计数器
 38     count = 0
 39     # 无限循环
 40     while True:
 41         try:
 42             count += 1
 43             # 显示等待商品信息加载完成
 44             WebDriverWait(browser, 1000).until(
 45                 EC.presence_of_all_elements_located(
 46                     (By.CLASS_NAME, 'gl-item')
 47                 )
 48             )
 49             # 将滚动条拉到最下面的位置，因为往下拉才能将这一页的商品信息全部加载出来
 50             browser.execute_script('document.documentElement.scrollTop=10000')
 51             # 随机延迟,等下元素全部刷新
 52             time.sleep(random.randint(1, 3))
 53             browser.execute_script('document.documentElement.scrollTop=0')
 54 
 55             # 开始提取信息,找到ul标签下的全部li标签
 56             lis = browser.find_elements_by_class_name('gl-item')
 57             # 遍历
 58             for li in lis:
 59                 # 名字
 60                 name = li.find_element_by_xpath('.//div[@class="p-name p-name-type-2"]//em').text
 61                 # 价格
 62                 price = li.find_element_by_xpath('.//div[@class="p-price"]//i').text
 63                 # 评论数
 64                 comment = li.find_elements_by_xpath('.//div[@class="p-commit"]//a')
 65                 if comment:
 66                     comment = comment[0].text
 67                 else:
 68                     comment = None
 69                 # 商铺名字
 70                 shop_name = li.find_elements_by_class_name('J_im_icon')
 71                 if shop_name:
 72                     shop_name = shop_name[0].text
 73                 else:
 74                     shop_name = None
 75                 # 商家类型
 76                 shop_type = li.find_elements_by_class_name('goods-icons')
 77                 if shop_type:
 78                     shop_type = shop_type[0].text
 79                 else:
 80                     shop_type = None
 81 
 82                 # 声明一个字典存储数据
 83                 data_dict = {}
 84                 data_dict['name'] = name
 85                 data_dict['price'] = price
 86                 data_dict['comment'] = comment
 87                 data_dict['shop_name'] = shop_name
 88                 data_dict['shop_type'] = shop_type
 89 
 90                 data_list.append(data_dict)
 91                 print(data_dict)
 92         except Exception as e:
 93             continue
 94 
 95         # 如果count==all_page就退出循环
 96         if count == all_page:
 97             break
 98         # 找到下一页的元素pn-next
 99         fp_next = browser.find_element_by_css_selector('a.fp-next')
100         # 点击下一页
101         fp_next.click()
102 
103 
104 def main():
105 
106     start_spider()
107     # 将数据写入jsonwenj
108     with open('data_json.json', 'a+', encoding='utf-8') as f:
109         json.dump(data_list, f, ensure_ascii=False, indent=4)
110     print('json文件写入完成')
111 
112     with open('data_csv.csv', 'w', encoding='utf-8', newline='') as f:
113         # 表头
114         title = data_list[0].keys()
115         # 声明writer
116         writer = csv.DictWriter(f, title)
117         # 写入表头
118         writer.writeheader()
119         # 批量写入数据
120         writer.writerows(data_list)
121     print('csv文件写入完成')
122 
123 
124 if __name__ == '__main__':
125 
126     main()
127     # 退出浏览器
128     browser.quit()

 

滑动验证

  1 from selenium import webdriver
  2 from selenium.webdriver.support.ui import WebDriverWait # 等待元素加载的
  3 from selenium.webdriver.common.action_chains import ActionChains  #拖拽
  4 from selenium.webdriver.support import expected_conditions as EC
  5 from selenium.common.exceptions import TimeoutException, NoSuchElementException
  6 from selenium.webdriver.common.by import By
  7 from PIL import Image
  8 import requests
  9 import time
 10 import re
 11 import random
 12 from io import BytesIO
 13 
 14 
 15 def merge_image(image_file,location_list):
 16     """
 17      拼接图片
 18     :param image_file:
 19     :param location_list:
 20     :return:
 21     """
 22     im = Image.open(image_file)
 23     im.save('code.jpg')
 24     new_im = Image.new('RGB',(260,116))
 25     # 把无序的图片 切成52张小图片
 26     im_list_upper = []
 27     im_list_down = []
 28     # print(location_list)
 29     for location in location_list:
 30         # print(location['y'])
 31         if location['y'] == -58: # 上半边
 32             im_list_upper.append(im.crop((abs(location['x']),58,abs(location['x'])+10,116)))
 33         if location['y'] == 0:  # 下半边
 34             im_list_down.append(im.crop((abs(location['x']),0,abs(location['x'])+10,58)))
 35 
 36     x_offset = 0
 37     for im in im_list_upper:
 38         new_im.paste(im,(x_offset,0))  # 把小图片放到 新的空白图片上
 39         x_offset += im.size[0]
 40 
 41     x_offset = 0
 42     for im in im_list_down:
 43         new_im.paste(im,(x_offset,58))
 44         x_offset += im.size[0]
 45     new_im.show()
 46     return new_im
 47 
 48 def get_image(driver,div_path):
 49     '''
 50     下载无序的图片  然后进行拼接 获得完整的图片
 51     :param driver:
 52     :param div_path:
 53     :return:
 54     '''
 55     time.sleep(2)
 56     background_images = driver.find_elements_by_xpath(div_path)
 57     location_list = []
 58     for background_image in background_images:
 59         location = {}
 60         result = re.findall('background-image: url\("(.*?)"\); background-position: (.*?)px (.*?)px;',background_image.get_attribute('style'))
 61         # print(result)
 62         location['x'] = int(result[0][1])
 63         location['y'] = int(result[0][2])
 64 
 65         image_url = result[0][0]
 66         location_list.append(location)
 67 
 68     print('==================================')
 69     image_url = image_url.replace('webp','jpg')
 70     # '替换url http://static.geetest.com/pictures/gt/579066de6/579066de6.webp'
 71     image_result = requests.get(image_url).content
 72     # with open('1.jpg','wb') as f:
 73     #     f.write(image_result)
 74     image_file = BytesIO(image_result) # 是一张无序的图片
 75     image = merge_image(image_file,location_list)
 76 
 77     return image
 78 
 79 def get_track(distance):
 80     '''
 81     拿到移动轨迹，模仿人的滑动行为，先匀加速后匀减速
 82     匀变速运动基本公式：
 83     ①v=v0+at
 84     ②s=v0t+(1/2)at²
 85     ③v²-v0²=2as
 86 
 87     :param distance: 需要移动的距离
 88     :return: 存放每0.2秒移动的距离
 89     '''
 90     # 初速度
 91     v=0
 92     # 单位时间为0.2s来统计轨迹，轨迹即0.2内的位移
 93     t=0.2
 94     # 位移/轨迹列表，列表内的一个元素代表0.2s的位移
 95     tracks=[]
 96     # 当前的位移
 97     current=0
 98     # 到达mid值开始减速
 99     mid=distance * 7/8
100 
101     distance += 10  # 先滑过一点，最后再反着滑动回来
102     # a = random.randint(1,3)
103     while current < distance:
104         if current < mid:
105             # 加速度越小，单位时间的位移越小,模拟的轨迹就越多越详细
106             a = random.randint(2,4)  # 加速运动
107         else:
108             a = -random.randint(3,5) # 减速运动
109 
110         # 初速度
111         v0 = v
112         # 0.2秒时间内的位移
113         s = v0*t+0.5*a*(t**2)
114         # 当前的位置
115         current += s
116         # 添加到轨迹列表
117         tracks.append(round(s))
118 
119         # 速度已经达到v,该速度作为下次的初速度
120         v= v0+a*t
121 
122     # 反着滑动到大概准确位置
123     for i in range(4):
124        tracks.append(-random.randint(2,3))
125     for i in range(4):
126        tracks.append(-random.randint(1,3))
127     return tracks
128 
129 
130 def get_distance(image1,image2):
131     '''
132       拿到滑动验证码需要移动的距离
133       :param image1:没有缺口的图片对象
134       :param image2:带缺口的图片对象
135       :return:需要移动的距离
136       '''
137     # print('size', image1.size)
138 
139     threshold = 50
140     for i in range(0,image1.size[0]):  # 260
141         for j in range(0,image1.size[1]):  # 160
142             pixel1 = image1.getpixel((i,j))
143             pixel2 = image2.getpixel((i,j))
144             res_R = abs(pixel1[0]-pixel2[0]) # 计算RGB差
145             res_G = abs(pixel1[1] - pixel2[1])  # 计算RGB差
146             res_B = abs(pixel1[2] - pixel2[2])  # 计算RGB差
147             if res_R > threshold and res_G > threshold and res_B > threshold:
148                 return i  # 需要移动的距离
149 
150 
151 
152 def main_check_code(driver, element):
153     """
154      拖动识别验证码
155     :param driver: 
156     :param element: 
157     :return: 
158     """
159     image1 = get_image(driver, '//div[@class="gt_cut_bg gt_show"]/div')
160     image2 = get_image(driver, '//div[@class="gt_cut_fullbg gt_show"]/div')
161     # 图片上 缺口的位置的x坐标
162 
163     # 2 对比两张图片的所有RBG像素点，得到不一样像素点的x值，即要移动的距离
164     l = get_distance(image1, image2)
165     print('l=',l)
166     # 3 获得移动轨迹
167     track_list = get_track(l)
168     print('第一步,点击滑动按钮')
169     ActionChains(driver).click_and_hold(on_element=element).perform()  # 点击鼠标左键，按住不放
170     time.sleep(1)
171     print('第二步,拖动元素')
172     for track in track_list:
173          ActionChains(driver).move_by_offset(xoffset=track, yoffset=0).perform()  # 鼠标移动到距离当前位置（x,y）　　　　　time.sleep(0.002)
174     # if l>100:
175 
176     ActionChains(driver).move_by_offset(xoffset=-random.randint(2,5), yoffset=0).perform()
177     time.sleep(1)
178     print('第三步,释放鼠标')
179     ActionChains(driver).release(on_element=element).perform()
180     time.sleep(5)
181 
182 
183 def main_check_slider(driver):
184     """
185     检查滑动按钮是否加载
186     :param driver: 
187     :return: 
188     """
189     while True:
190         try :
191             driver.get('http://www.cnbaowen.net/api/geetest/')
192             element = WebDriverWait(driver, 30, 0.5).until(EC.element_to_be_clickable((By.CLASS_NAME, 'gt_slider_knob')))
193             if element:
194                 return element
195         except TimeoutException as e:
196             print('超时错误，继续')
197             time.sleep(5)
198 
199 
200 if __name__ == '__main__':
201     try:
202         count = 6  # 最多识别6次
203         driver = webdriver.Chrome()
204         # 等待滑动按钮加载完成
205         element = main_check_slider(driver)
206         while count > 0:
207             main_check_code(driver,element)
208             time.sleep(2)
209             try:
210                 success_element = (By.CSS_SELECTOR, '.gt_holder .gt_ajax_tip.gt_success')
211                 # 得到成功标志
212                 print('suc=',driver.find_element_by_css_selector('.gt_holder .gt_ajax_tip.gt_success'))
213                 success_images = WebDriverWait(driver, 20).until(EC.presence_of_element_located(success_element))
214                 if success_images:
215                     print('成功识别！！！！！！')
216                     count = 0
217                     break
218             except NoSuchElementException as e:
219                 print('识别错误，继续')
220                 count -= 1
221                 time.sleep(2)
222         else:
223             print('too many attempt check code ')
224             exit('退出程序')
225     finally:
226         driver.close()

 

posted @ 2019-06-18 22:16  鸡肉串  阅读(322)  评论(0)    收藏  举报