【Python学习】爬虫源码

1、在巨人的肩膀上，结合网上的资源，梳理出来的。

2、其中应用了几个常用的包，requests、re等，

3、注意创建多级文件夹要用--makesdir，创建单级文件用--mkdir

# 导入相应的包
# 请求网页
import requests
# 正则解析网页
import re
# 告诉服务，自己的身份，
import time

import os

# 函数请求的头部信息
headers = {
    "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68'
}

############################################  开始编写程序  ############################################
# 定义轮询的页面数
polling_page_num = int(input("请输入主页页码轮询的数量:"))

############################################  函数区  ############################################

# 下载界面html代码的函数
def download_page(url, html_encode='utf-8', *args, **kwargs):
    """
    下载界面html代码的函数
    :param url:需要下载网页代码的链接
    :param html_encode:网页的解码方式。默认是“utf-8”
    :param args:
    :param kwargs:
    :return:返回值为该页面的html代码
    """
    headers = {
        "user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68'}
    r = requests.get(url, headers=headers)
    r.encoding = html_encode
    response_status = r.status_code
    return r.text, response_status


# 分析主网页并且返回已经处理后的链接列表
def anasy_main_page(html, *args, **kwargs):
    ex = '<a href="(.*?)" title=".*?"><img alt=".*?" src=".*?"><i>.*?</i></a>'
    # ex = '<a href="(.*?)" title=.*?><img alt=.*? src=.*?><i>.*?</i></a>'
    test_src_list = re.findall(ex, html, re.S)
    new_src_list = test_src_list[1:31]
    li_piclink = []
    for pic_link in new_src_list:
        new_pic_link = '链接' + pic_link
        li_piclink.append(new_pic_link)
    return li_piclink


# 分析副网页,返回图片的下载地址
def anasy_Secondary_page(Secondary_html):
    """
    :param Secondary_html:
    :return: 一个元组，
            dir_name --文件夹名称
            pic_link --- 链接
    """
    ex_link = '<img  alt=".*?" src="(.*?)"  />'
    ex_name = '<h1>(.*?)</h1>'  # 需要修改的地方
    pic_link = re.findall(ex_link, Secondary_html, re.S)[0]
    dir_name = re.findall(ex_name, Secondary_html, re.S)[0]
    return dir_name, pic_link


# 创建文件夹
def create_folder(dir_name):
    dir_name_cl = "".join(dir_name.split())
    dir_name = dir_name_cl
    if not os.path.exists(dir_name):
        os.mkdir(dir_name)
    return dir_name


# 下载图片
def down_pic(dir_name, pic_link):
    """
    :param dir_name:
    :param pic_link:
    :return:
    """
    img_data = requests.get(url=pic_link, headers=headers).content
    img_name = pic_link.split('/')[-1]
    imgPath = dir_name + '/' + img_name
    with open(imgPath, 'wb') as f:
        f.write(img_data)
    return


# 网页主页生成器
def create_main_url(url_num):
    url_ys = '子链接'
    mian_url_list = []
    if url_num > 1:
        start_num = 2
    else:
        start_num = 1

    for url_n in range(start_num, url_num + 1):
        if url_n != 1:
            url = url_ys + 'index_%d.html'
            new_url = format(url % url_n)
        else:
            new_url = url_ys
        mian_url_list.append(new_url)
    return mian_url_list


# 子网页主页生成器
def create_sec_url(url, url_num, *args, **kwargs):
    """
    :param url:
    :param url_num:
    :return:
    """
    sec_url_list = []
    for url_n in range(1, url_num + 1):
        if url_n != 1:
            # new_url = url + '_'+str(url_n)+'.html'
            begin = url.find("h")
            end = url.rfind(".")
            find_url = url[begin:end]
            new_url = find_url + '_' + str(url_n) + '.html'
        else:
            new_url = url
        sec_url_list.append(new_url)
    return sec_url_list


# 下载日志生成
def create_log(log_content):
    """
    下载日志生成函数
    :param log_content: 写入log的内容
    :return: 无
    """
    with open("log.txt", "a") as file:
        file.write(log_content)
    return


# 页面记录器
def page_record(page_num=0, *args, **kwargs):
    with open("page_record.txt", "w+") as file:
        file.write(page_num)
    return


# 读取配置
def page_read():
    with open("page_record.txt", "r") as file:
        r_page_num = file.readline()
    return r_page_num


############################################  爬虫工作区  ############################################

n_yema = int(page_read())
# print(n_yema)
if polling_page_num > 361:
    print("您输入的超出轮询范围，请重新输入!")
elif polling_page_num > n_yema:
    end_page_num = polling_page_num
    print("主程序即将进行")

    # 生成主网页
    mian_url_list_ys = create_main_url(end_page_num)
    mian_url_list = mian_url_list_ys[int(n_yema)-1:int(end_page_num)+1]


    for url in mian_url_list:
        n_yema = n_yema + 1
        sec_url_li = anasy_main_page(download_page(url)[0])  # 分析主链接，获得套图列表
        print(len(sec_url_li), sec_url_li)
        log_mian_start = "*" * 15 + "第" + str(n_yema) + "页，开始下载-->" + url + "*" * 15
        print(log_mian_start)  # 某些页面开始下载的提示
        n_tao = 0
        for url_sec in sec_url_li[0:31]:
            n_tao = n_tao + 1
            dir_name = anasy_Secondary_page(download_page(url_sec, html_encode="utf-8")[0])[0]  # 分析指定的套图链接，获得套图的名称


            print("*" * 15 + "第" + str(n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "--" + "开始下载" + "*" * 15)
            dir_name_sj = create_folder(dir_name)
            sec_url_list = create_sec_url(url_sec, 60)
            m = 0
            for pic_link in sec_url_list:
                m = m + 1
                page_text, response_status_pic = download_page(pic_link)
                if response_status_pic == 200:
                    donw_pic_link = anasy_Secondary_page(page_text)[1]  # 分析指定的套图链接，获得图片链接
                    down_pic(dir_name_sj, donw_pic_link)
                    print("第" + str(m) + "张图片下成功", donw_pic_link)
                    time.sleep(1)
                else:
                    continue

            print("第" + str(n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "全部图片下载完毕" + "\n")
            log_text = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + "---" + "第" + str(
                n_yema) + "页，第" + str(n_tao) + "套图--" + dir_name + "已经下载完毕" + "\n"
            create_log(log_content=log_text)
        log_main_end = "*" * 10 + "第" + str(n_yema) + "页，下载完成-->" + url + "*" * 10 + "\n\n"
        print(log_main_end)
        # 要记录已经下载的主页的页面页码（n_yema）
        page_record(str(n_yema))