第六次作业

一、作业内容

作业①:

要求：

        用requests和BeautifulSoup库方法爬取豆瓣电影Top250数据。
        每部电影的图片，采用多线程的方法爬取，图片名字为电影名
        了解正则的使用方法

候选网站：豆瓣电影：https://movie.douban.com/top250

代码：

from bs4 import BeautifulSoup
from bs4 import UnicodeDammit
import urllib.request
import threading

def imageSpider(start_url):
    global thread
    global count
    try:
        urls = []
        req = urllib.request.Request(start_url, headers=headers)
        data = urllib.request.urlopen(req)
        data = data.read()
        dammit = UnicodeDammit(data, ["utf-8", "gbk"])
        data = dammit.unicode_markup
        soup = BeautifulSoup(data, "html.parser")
        list = soup.select('ol[class="grid_view"] li')
        for li in list:
            rank=li.select('div[class="pic"] em[class=""]')[0].text.replace("em",'').replace("\n",'')
            fn=li.select('div[class="info"] div a span ')[0].text.replace("span",'').replace("\n",'')
            info_l=li.select('div[class="bd"] p')[0].text.replace("p",'').replace("\n",'').replace(" ",'')
            info_f =li.select('div[class="bd"] br')[0].text.replace("br",'').replace("\n",'').replace(" ",'')
            comment=li.select('div[class="bd"] div[class="star"] span[class="rating_num"]')[0].text.replace("span",'').replace("\n",'').replace(" ",'')#评价信息
            audi=li.select('div[class="star"] span')[3].text.replace("span",'').replace("\n",'').replace(" ",'')
            print(rank,fn,info_l,info_f,comment,audi)
        images1 = soup.select("img")
        for image in images1:
            try:
                src = image["src"]
                url = urllib.request.urljoin(start_url, src)
                if url not in urls:
                    urls.append(url)
                    print(url)
                    count = count + 1
                    T = threading.Thread(target=download, args=(url, count))
                    T.setDaemon(False)
                    T.start()
                    thread.append(T)
            except Exception as err:
                print(err)
    except Exception as err:
        print(err)


def download(url, count):
    try:
        count=count+1
        if (url[len(url) - 4] == "."):
            ext = url[len(url) - 4:]
        else:
            ext = ""
        req = urllib.request.Request(url, headers=headers)
        data = urllib.request.urlopen(req, timeout=100)
        data = data.read()
        fobj = open("photo\\" + str(count) + ext, "wb")
        fobj.write(data)
        fobj.close()
        print("downloaded " + str(count) + ext)
    except Exception as err:
        print(err)
start_url = "https://movie.douban.com/top250"
headers = {"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre"}
count = 0
thread = []


imageSpider(start_url)
for t in thread:
    t.join()
print("the End")

结果：

心得：

这次作业主要是回顾上半学期的bs和request库爬虫，相比于之前我主要是在静态网页定位上有所遗忘，进行了重新学习。

作业②:

要求：

    熟练掌握 scrapy 中 Item、Pipeline 数据的序列化输出方法；Scrapy+Xpath+MySQL数据库存储技术路线爬取科软排名信息
    爬取科软学校排名，并获取学校的详细链接，进入下载学校Logo存储、获取官网Url、院校信息等内容。

候选网站：https://www.shanghairanking.cn/rankings/bcur/2020

代码：

items.py:

import scrapy
class UniversityrankingItem(scrapy.Item):
    sNo=scrapy.Field()
    schoolName=scrapy.Field()
    city=scrapy.Field()
    officalUrl=scrapy.Field()
    info=scrapy.Field()

pipelines.py

import sqlite3
class UniversityrankingPipeline:


    class SpiderPipeline:
        def open_spider(self, MySpider):
            self.con = sqlite3.connect("University.db")
            self.cursor = self.con.cursor()
            try:
                self.cursor.execute("drop table University")
            except:
                pass
            sql = "create table University(sNo int,schoolName varchar(32),city varchar(16),officalUrl varchar(64),"\
                    "info text,);"
            self.cursor.execute(sql)
            #self.cursor.execute("delete from University")

        def close_spider(self, MySpider):
            if self.opened: self.con.commit()

            self.con.close()
            self.opened = False
            print("closed")

        def process_item(self, item, spider):
            try:
                print(item["sNo"])
                print(item["schoolName"])
                print(item["city"])
                print(item["officalUrl"])
                print(item["info"])
                print()
                self.cursor.execute("insert into University(sNo,schoolName,city,officalUrl,info) "
                    "values(?,?,?,?,?)",
                    (sNo, schoolName, city, officalUrl, info))

            except Exception as err:
                print(err)
            return item

结果：

由于一直没能实现将图片存入数据库中，我最后放弃了这一想法，只存储了排名、校名、城市、官网、信息五项。

感想：

结果到最后还是没能用上pymysql啊......
这一题仿照了原来的架构，在xpath上有所遗忘，但总体完成，还是比较清晰的。

作业③：

要求：

    熟练掌握 Selenium 查找HTML元素、爬取Ajax网页数据、等待HTML元素加载、网页跳转等内容。
    使用Selenium框架+ MySQL数据库存储技术模拟登录慕课网，并获取学生自己账户中已学课程的信息并保存在MYSQL中。
    其中模拟登录账号环节需要录制gif图。

候选网站： 中国mooc网：https://www.icourse163.org

代码：

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import sqlite3
import time

class MOOC:
    def Load_in(self):
        time.sleep(1)
        user = self.driver.find_element_by_xpath('//*[@id="j-topnav"]/div')
        user.click()
        time.sleep(1)
        way = self.driver.find_element_by_xpath('//div[@class="ux-login-set-scan-code_ft"]/span')
        way.click()
        time.sleep(1)
        telephone = self.driver.find_element_by_xpath('//ul[@class="ux-tabs-underline_hd"]/li[2]')
        telephone.click()
        time.sleep(1)
        frame = self.driver.find_element_by_xpath(
            "/html/body/div[13]/div[2]/div/div/div/div/div/div[1]/div/div[1]/div[2]/div[2]/div/iframe")
        self.driver.switch_to.frame(frame)
        self.driver.find_element_by_xpath('//input[@type="tel"]').send_keys('15750908582')
        time.sleep(1)
        self.driver.find_element_by_xpath('//input[@class="j-inputtext dlemail"]').send_keys('wuyinhang817000')
        time.sleep(1)
        load_in = self.driver.find_element_by_xpath('//*[@id="submitBtn"]')
        load_in.click()

    def MyClass(self):
        time.sleep(1)
        myclass = self.driver.find_element_by_xpath('//*[@id="j-indexNav-bar"]/div/div/div/div/div[7]/div[3]/div')
        myclass.click()
        self.all_spider()

    def all_spider(self):
        time.sleep(1)
        self.spider()
        time.sleep(1)
        try:
            self.driver.find_element_by_xpath(
                '//ul[@class="ux-pager"]/li[@class="ux-pager_btn ux-pager_btn__next"]/a[@class="th-bk-disable-gh"]')
        except Exception:
            self.driver.find_element_by_xpath(
                '//ul[@class="ux-pager"]/li[@class="ux-pager_btn ux-pager_btn__next"]/a[@class="th-bk-main-gh"]').click()
            self.all_spider()

    def spider(self):
        id = 0
        time.sleep(1)
        lis = self.driver.find_elements_by_xpath('//div[@class="course-card-wrapper"]')
        for li in lis:
            time.sleep(1)
            li.click()
            window = self.driver.window_handles
            self.driver.switch_to.window(window[1])
            time.sleep(1)
            window = self.driver.window_handles
            self.driver.switch_to.window(window[2])
            time.sleep(1)
            id += 1
            course = self.driver.find_element_by_xpath(
                '//*[@id="g-body"]/div[1]/div/div[3]/div/div[1]/div[1]/span[1]').text
            teacher = self.driver.find_element_by_xpath('//*[@id="j-teacher"]//h3[@class="f-fc3"]').text
            collage = self.driver.find_element_by_xpath('//*[@id="j-teacher"]/div/a/img').get_attribute('alt')
            process = self.driver.find_element_by_xpath('//*[@id="course-enroll-info"]/div/div[1]/div[2]/div[1]').text
            count = self.driver.find_element_by_xpath('//*[@id="course-enroll-info"]/div/div[2]/div[1]/span').text
            brief = self.driver.find_element_by_xpath('//*[@id="j-rectxt2"]').text
            self.cursor.execute("insert into mooc(id, course, teacher, collage, count, process, brief) "
                                "values( ?,?,?,?,?,?,?)",
                                (id, course, teacher, collage, count, process, brief))

            time.sleep(1)
            self.driver.close()
            self.driver.switch_to.window(window[1])
            time.sleep(1)
            self.driver.close()
            self.driver.switch_to.window(window[0])

    def start(self):
        self.con = sqlite3.connect("mooc.db")
        self.cursor = self.con.cursor()
        try:
            self.cursor.execute("drop table mooc")
        except:
            pass
        sql = 'create table mooc(id int,course varchar(32),teacher varchar(16),collage varchar(32),count varchar(64),' \
               'process varchar(64),brief text);'
        self.cursor.execute(sql)
        #self.cursor.execute("delete from mooc")

    def stop(self):
        try:
            self.con.commit()
            self.con.close()
        except Exception as err:
            print(err)

    def executespider(self, url):
        chrome_options = Options()
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.driver.get(url)
        self.start()
        self.Load_in()
        self.MyClass()
        self.stop()

def main():
        url = 'https://www.icourse163.org/'
        spider = MOOC()
        spider.executespider(url)

if __name__ == '__main__':
        main()

结果：

心得：

使用了上次的selenium框架，这次成功的完成了模拟登录账号，删掉了选择界面。
有时会爬到一半报错：selenium.common.exceptions.WebDriverException: Message: chrome not reachable
有时正常，尚在研究之中。

写在最后：

一学期的学习就这样结束了。本人并不是特别聪明的类型，作业做得马马虎虎，每次做作业都深刻的体会到了人和人的差距......
但还是要感谢吴老师、陈助教以及班里的各位大佬的帮助，大家对我相当宽容，给了我很多帮助，让我受益良多。
对于一门课、一种技术而言，兴趣不一定和成果成严格正比，但一定存在正相关。
愿我们以后还能把兴趣挥洒在每一个角落。再次感谢！