第五次作业

作业①:

要求：

候选网站：http://www.jd.com/

关键词：学生自由选择

1）爬取京东实验：

2.编写代码:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import urllib.request
import threading
import sqlite3
import os
import datetime
from selenium.webdriver.common.keys import Keys
import time
class MySpider:
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.361"}
    imagePath = "download"
    def startUp(self, url, key):
        # # Initializing Chrome browser
        chrome_options = Options()
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        # Initializing variables
        self.threads = []
        self.No = 0
        self.imgNo = 0
        # Initializing database
        try:
            self.con = sqlite3.connect("computer.db")
            self.cursor = self.con.cursor()
            try:
                # 如果有表就删除
                self.cursor.execute("drop table phones")
            except:
                pass
            try:
                sql = "create  table  computer  (mNo  varchar(32) primary key, mMark varchar(256),mPrice varchar(32),mNote varchar(1024),mFile varchar(256))"
                self.cursor.execute(sql)
            except:
                pass
        except Exception as err:
            print(err)
        # Initializing images folder
        try:
            if not os.path.exists(MySpider.imagePath):
                os.mkdir(MySpider.imagePath)
            images = os.listdir(MySpider.imagePath)
            for img in images:
                s = os.path.join(MySpider.imagePath, img)
                os.remove(s)
        except Exception as err:
            print(err)
        self.driver.get(url)
        keyInput = self.driver.find_element_by_id("key")
        keyInput.send_keys(key)
        keyInput.send_keys(Keys.ENTER)
    def closeUp(self):
        try:
            self.con.commit()
            self.con.close()
            self.driver.close()
        except Exception as err:
            print(err);
    def insertDB(self, mNo, mMark, mPrice, mNote, mFile):
        try:
            sql = "insert into phones (mNo,mMark,mPrice,mNote,mFile) values (?,?,?,?,?)"
            self.cursor.execute(sql, (mNo, mMark, mPrice, mNote, mFile))
        except Exception as err:
            print(err)
    def showDB(self):
        try:
            con = sqlite3.connect("phones.db")
            cursor =con.cursor()
            print("%-8s%-16s%-8s%-16s%s"%("No", "Mark", "Price", "Image", "Note"))
            cursor.execute("select mNo,mMark,mPrice,mFile,mNote from phones  order by mNo")
            rows = cursor.fetchall()
            for row in rows:
                print("%-8s %-16s %-8s %-16s %s" % (row[0], row[1], row[2], row[3],row[4]))
            con.close()
        except Exception as err:
            print(err)
    def download(self, src1, src2, mFile):
        data = None
        if src1:
            try:
                req = urllib.request.Request(src1, headers=MySpider.headers)
                resp = urllib.request.urlopen(req, timeout=10)
                data = resp.read()
            except:
                pass
        if not data and src2:
            try:
                req = urllib.request.Request(src2, headers=MySpider.headers)
                resp = urllib.request.urlopen(req, timeout=10)
                data = resp.read()
            except:
                pass
        if data:
            print("download begin", mFile)
            fobj = open(MySpider.imagePath + "\\" + mFile, "wb")
            fobj.write(data)
            fobj.close()
            print("download finish", mFile)
    def processSpider(self):
        try:
            time.sleep(1)
            print(self.driver.current_url)
            lis =self.driver.find_elements_by_xpath("//div[@id='J_goodsList']//li[@class='gl-item']")
            for li in lis:
            # We find that the image is either in src or in data-lazy-img attribute
                try:
                    src1 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("src")
                except:
                    src1 = ""
                try:
                    src2 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("data-lazy-img")
                except:
                    src2 = ""
                try:
                    price = li.find_element_by_xpath(".//div[@class='p-price']//i").text
                except:
                    price = "0"
                try:
                    note = li.find_element_by_xpath(".//div[@class='p-name p-name-type-2']//em").text
                    mark = note.split(" ")[0]
                    mark = mark.replace("爱心东东\n", "")
                    mark = mark.replace(",", "")
                    note = note.replace("爱心东东\n", "")
                    note = note.replace(",", "")
                except:
                    note = ""
                    mark = ""
                try:
                    price = li.find_element_by_xpath(".//div[@class='p-price']//i").text
                except:
                    price = "0"
                try:
                    note = li.find_element_by_xpath(".//div[@class='p-name p-name-type-2']//em").text
                    mark = note.split(" ")[0]
                    mark = mark.replace("爱心东东\n", "")
                    mark = mark.replace(",", "")
                    note = note.replace("爱心东东\n", "")
                    note = note.replace(",", "")
                except:
                    note = ""
                    mark = ""
            self.No = self.No + 1
            no = str(self.No)
            while len(no) < 6:
                no = "0" + no
            print(no, mark, price)
            if src1:
                src1 = urllib.request.urljoin(self.driver.current_url, src1)
                p = src1.rfind(".")
                mFile = no + src1[p:]
            elif src2:
                src2 = urllib.request.urljoin(self.driver.current_url, src2)
                p = src2.rfind(".")
                mFile = no + src2[p:]
            if src1 or src2:
                T = threading.Thread(target=self.download, args=(src1, src2, mFile))
                T.setDaemon(False)
                T.start()
                self.threads.append(T)
            else:
                mFile = ""
            self.insertDB(no, mark, price, note, mFile)
            # 取下一页的数据，直到最后一页
            try:
                self.driver.find_element_by_xpath("//span[@class='p-num']//a[@class='pn-next disabled']")
            except:
                nextPage = self.driver.find_element_by_xpath("//span[@class='p-num']//a[@class='pn-next']")
                time.sleep(10)
                nextPage.click()
                self.processSpider()
        except Exception as err:
            print(err)
    def executeSpider(self, url, key):
        starttime = datetime.datetime.now()
        print("Spider starting......")
        self.startUp(url, key)
        print("Spider processing......")
        self.processSpider()
        print("Spider closing......")
        self.closeUp()
        for t in self.threads:
            t.join()
        print("Spider completed......")
        endtime = datetime.datetime.now()
        elapsed = (endtime - starttime).seconds
        print("Total ", elapsed, " seconds elapsed")
url = "http://www.jd.com"
spider = MySpider()
while True:
    print("1.爬取")
    print("2.显示")
    print("3.退出")
    s = input("请选择(1,2,3):")
    if s == "1":
        spider.executeSpider(url, "笔记本")
        continue
    elif s == "2":
        spider.showDB()
        continue
    elif s == "3":
        break

爬取结果：

不知道为什么会混入书本的图片

2）心得体会

这主要是ppt中代码的复现。为后面的两个作业奠定框架

作业②

• 要求：

  • 熟练掌握 Selenium 查找HTML元素、爬取Ajax网页数据、等待HTML元素等内容。
  • 使用Selenium框架+ MySQL数据库存储技术路线爬取“沪深A股”、“上证A股”、“深证A股”3个板块的股票数据信息。

• 候选网站：东方财富网：http://quote.eastmoney.com/center/gridlist.html#hs_a_board

  1）爬取股票信息实验

  代码部分：

  from selenium import webdriver
  from selenium.webdriver.chrome.options import Options
  import pymysql
  import datetime
  import time
  
  
  class MySpider:
      headers = {
          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.361"
      }
      count = 1
  
      def startUp(self, url):
  
          chrome_options = Options()
          chrome_options.add_argument('--headless')
          chrome_options.add_argument('--disable-gpu')
          self.driver = webdriver.Chrome(chrome_options=chrome_options)
          # Initializing variables
          self.threads = []
          self.No = 0
          self.imgNo = 0
          try:
              self.con = pymysql.connect(host="127.0.0.1", port=3306, user="root", passwd="123456", db="mydb",
                                         charset="utf8")
              self.cursor = self.con.cursor(pymysql.cursors.DictCursor)
              self.opened = True
              self.page_num = 1
          except Exception as err:
              print(err)
              self.opened = False
          self.driver.get(url)
  
      def closeUp(self):
          try:
              if (self.opened):
                  self.con.commit()
                  self.con.close()
                  self.opened = False
              self.driver.close()
              print("爬取完毕，closed")
          except Exception as err:
              print("关闭数据库失败", err)
  
      def insertDB(self, Position, id, No, name, price, edu, fudu, cjl, cje, zf, highest,
                   lowest, today, yestoday):
          try:
              self.cursor.execute(
                  "insert into my_stock(Position,id,No,name,price,edu,fudu,cjl,cje,zf,highest,lowest,today,yestoday) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
                  (str(self.count), id, No, name, price, edu, fudu, cjl, cje, zf, highest,
                   lowest, today, yestoday))
          except Exception as err:
              print("插入失败", err)
  
      def processSpider(self):
          try:
              time.sleep(1)
              print(self.driver.current_url)
              tr_list = self.driver.find_elements_by_xpath("//*[@id='table_wrapper-table']/tbody/tr")
              for tr in tr_list:
                  id = tr.find_element_by_xpath('./td[1]').text
                  No = tr.find_element_by_xpath('./td[2]').text
                  name = tr.find_element_by_xpath('./td[3]').text
                  price = tr.find_element_by_xpath('./td[5]').text
                  edu = tr.find_element_by_xpath('./td[6]').text
                  fudu = tr.find_element_by_xpath('./td[7]').text
                  cjl = tr.find_element_by_xpath('./td[8]').text
                  cje = tr.find_element_by_xpath('./td[9]').text
                  zf = tr.find_element_by_xpath('./td[10]').text
                  highest = tr.find_element_by_xpath('./td[11]').text
                  lowest = tr.find_element_by_xpath('./td[12]').text
                  today = tr.find_element_by_xpath('./td[13]').text
                  yestoday = tr.find_element_by_xpath('./td[14]').text
                  self.insertDB(str(self.count), id, No, name, price, edu, fudu, cjl, cje,
                                zf, highest, lowest, today, yestoday)
                  self.count += 1
              try:
                  self.driver.find_element_by_xpath(
                      '//*[@id="main-table_paginate"]/a[@class="next paginate_button disabled"]')
              except:
                  next_page = self.driver.find_element_by_xpath(
                      '//*[@id="main-table_paginate"]/a[@class="next paginate_button"]')
                  time.sleep(3)
                  next_page.click()
                  # time.sleep(5)
                  if (self.page_num < 3):
                      self.page_num += 1
                      self.processSpider()
          except Exception as err:
              print(err)
  
  
  myspider = MySpider()
  data = {
      "沪深A股": "gridlist.html#hs_a_board",
      "上圳A股": "gridlist.html#sh_a_board",
      "深圳A股": "gridlist.html#sz_a_board"
  }
  starttime = datetime.datetime.now()
  for key in data.keys():
      url = "http://quote.eastmoney.com/center/"
      print("正在爬取" + key + "板块的股票")
      url = url + data[key]
      print("开始爬虫...")
      myspider.startUp(url)
      print("正在爬虫...")
      myspider.processSpider()
      myspider.closeUp()
  endtime = datetime.datetime.now()
  total_time = endtime - starttime
  print("结束爬虫," + "一共耗时" + str(total_time) + "秒")
  print("一共耗时" + str(myspider.count - 1) + "条数据");
  

爬取结果：

2）心得体会：

这个还是和作业1类似，再用以前爬股票的相关内容按照1中单框架就可以了。但是设计数据库的时候设置主键会在爬取过程中冲突，就没有设置主键了。

作业③

1）爬取MOOC实验

代码部分：

from selenium import webdriver
from selenium.webdriver import ChromeOptions
from time import sleep
import pymysql
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
driver = webdriver.Chrome(options=option)

driver.get("https://www.icourse163.org/")
driver.maximize_window()
sleep(2)

driver.find_element_by_xpath('//div[@class="unlogin"]//a[@class="f-f0 navLoginBtn"]').click()   #登录或注册
sleep(2)
driver.find_element_by_class_name('ux-login-set-scan-code_ft_back').click()              #其他登录方式
sleep(2)
driver.find_element_by_xpath("//ul[@class='ux-tabs-underline_hd']//li[@class='']").click()
sleep(2)
driver.switch_to.frame(driver.find_element_by_xpath("//div[@class='ux-login-set-container']//iframe"))
driver.find_element_by_xpath('//input[@id="phoneipt"]').send_keys("18650084388")        #输入账号
sleep(2)
driver.find_element_by_xpath('//input[@placeholder="请输入密码"]').send_keys("wuqilin2000420")   #输入密码
sleep(2)
driver.find_element_by_xpath('//div[@class="f-cb loginbox"]//a[@id="submitBtn"]').click()  #点击登录
sleep(3)
driver.find_element_by_xpath(
    '//div[@class="u-baseinputui"]/input[@class="j-textarea inputtxt"]').send_keys("医学")    #输入要找的课程
sleep(2)
driver.find_element_by_xpath('//div[@class="u-search-icon"]/span[@class="u-icon-search2 j-searchBtn"]').click()   #点击搜索
sleep(2)
conn = None
cursor = None
conn = pymysql.Connect(host='127.0.0.1', port=3306, user='root', password='123456', db='mydb', charset='utf8')
cursor = conn.cursor()
id=0
while True:
    sleep(2)
    divs = driver.find_elements_by_xpath('//div[@class="m-course-list"]/div/div')
    for i in range(2):
        try:
            div = driver.find_elements_by_xpath('//div[@class="m-course-list"]/div/div')[i + 1]
            div.click()
            sleep(3)
            current_window = driver.window_handles[-1]   #切换到最新打开的页面
            driver.switch_to.window(current_window)
            sleep(2)
            id += 1
            course = driver.find_element_by_xpath('//span[@class="course-title f-ib f-vam"]').text
            process = driver.find_element_by_xpath(
                '//div[@class="course-enroll-info_course-info_term-info_term-time"]/span[2]').text
            college = driver.find_element_by_xpath('//*[@id="j-teacher"]/div/a/img').get_attribute("alt")
            count = driver.find_element_by_xpath(
                '//span[@class="course-enroll-info_course-enroll_price-enroll_enroll-count"]').text
            brief = driver.find_element_by_xpath('//*[@id="j-rectxt2"]').text
            teacher = driver.find_element_by_xpath('//div[@class="cnt f-fl"][1]/h3').text
            team = ""
            teas = driver.find_elements_by_xpath('//div[@class="um-list-slider_con"]/div')
            if len(teas)>1:
                for tea in teas:
                    team = team + tea.find_element_by_xpath('.//div[@class="cnt f-fl"]/h3').text   #教师拼接
            else:
                team=teacher
            print(course, college, teacher, team, process, brief)
            try:
                cursor.execute('insert into my_mooc values("%s","%s","%s","%s","%s","%s","%s","%s")' %
                               (id, course, college, teacher, team, count, process, brief))  # 插入数据
                conn.commit()
            except:
                conn.rollback()
            driver.close()
            sleep(2)
            previous_window = driver.window_handles[0]   #切换回最开始打开的界面
            sleep(2)
            driver.switch_to.window(previous_window)
            sleep(2)
            try:
                driver.find_element_by_xpath(
                    '//li[@class="ux-pager_btn ux-pager_btn__next"]/a[@class="th-bk-disable-gh"]')
            except:
                driver.find_element_by_xpath(
                    '//li[@class="ux-pager_btn ux-pager_btn__next"]/a[@class="th-bk-main-gh"]').click()
                sleep(3)
        except Exception as e:
            print(e)
driver.quit()
cursor.close()
conn.close()

爬取结果：

爬到一半就报错了

页面卡在了教师介绍页面就不动了，暂时不知道咋解决