爬取音乐资源

1说明

目标地址：http://www.luoo.net/music/中指定期数的音乐资源

例如：http://www.luoo.net/vol/index/1368

2知识点

requests：比urllib更简单

faker：模拟UA请求

queue：队列，注意Python3.x中是import queue

类（多线程）：Python类定义和继承

快捷键：Ctrl + / 注释(取消注释)选择的行

3代码

import os
import requests
from bs4 import BeautifulSoup
import random
from faker import Factory
import queue
import threading

# 生成UA（）
fake = Factory.create()
headers = {
    'Connection': 'keep-alive',
    'User-Agent': fake.user_agent()
}

# 爬取网址
luoo_site = 'http://www.luoo.net/vol/index/'
# 点击某一个资源，从Network中获取下载路径
luoo_site_mp3 = 'http://mp3-cdn2.luoo.net/low/luoo/radio%s/%s.mp3'

# 配置可用的代理IP
proxy_ips = [
    '183.129.151.130'
]


# 生成随机IP
def random_proxies():
    ip_index = random.randint(0, len(proxy_ips) - 1)
    res = {'http': proxy_ips[ip_index]}
    return res


# FIXME 特殊字符处理
def fix_characters(s):
    for c in ['<', '>', ':', '"', '/', '\\', '|', '?', '*']:
        s = s.replace(c, '')
    return s

# 爬虫类定义，继承线程类threading.Thread
class LuooSpider(threading.Thread):
    # 实例化一个类的时候，一定会执行的方法
    def __init__(self, url, vols, queue=None):
        print('LuooSpider __init__ begin')
        threading.Thread.__init__(self)
        self.url = url
        self.queue = queue
        self.vol = '1'
        self.vols = vols
        print('LuooSpider __init__ end')

    # 线程的执行方法
    def run(self):
        for vol in self.vols:
            self.spider(vol)
        print('crawl end')

    # 爬取第vol期内容
    def spider(self, vol):
        url = luoo_site + vol
        print('crawling: ' + url)
        # res = requests.get(url, proxies=random_proxies()) # 这个有问题
        res = requests.get(url)
        # print(res.content)

        # 右键查看源代码，然后可以找到相应元素（或者F12）
        soup = BeautifulSoup(res.content, 'html.parser')
        title = soup.find('span', attrs={'class': 'vol-title'}).text  # 获取期刊标题
        cover = soup.find('img', attrs={'class': 'vol-cover'})['src']  # 获取期刊封面
        desc = soup.find('div', attrs={'class': 'vol-desc'})  # 获取期刊描述
        track_names = soup.find_all('a', attrs={'class': 'trackname'})  # 歌曲名称列表
        track_count = len(track_names)

        # 获取节目清单
        tracks = []
        for track in track_names:
            # 12期前的音乐编号1~9是1位（如：1~9），之后的都是2位 1~9会在左边垫0（如：01~09）
            _id = str(int(track.text[:2])) if (int(vol) < 12) else track.text[:2]
            _name = fix_characters(track.text[4:])
            tracks.append({'id': _id, 'name': _name})

        # 期刊内容
        phases = {
            'phase': vol,  # 期刊编号
            'title': title,  # 期刊标题
            'cover': cover,  # 期刊封面
            'desc': desc,  # 期刊描述
            'track_count': track_count,  # 节目数
            'tracks': tracks  # 节目清单(节目编号，节目名称)
        }

        print("phases：" + str(phases))
        self.queue.put(phases)


# 下载类定义，继承线程类threading.Thread
class LuooDownloader(threading.Thread):
    def __init__(self, url, dist, queue=None):
        threading.Thread.__init__(self)
        self.url = url
        self.queue = queue
        self.dist = dist
        self.__counter = 0

    def run(self):
        while True:
            if self.queue.qsize() <= 0:
                pass
            else:
                phases = self.queue.get()
                self.download(phases)

    def download(self, phases):
        for track in phases['tracks']:
            file_url = self.url % (phases['phase'], track['id'])

            local_file_dict = '%s/%s' % (self.dist, phases['phase'])
            if not os.path.exists(local_file_dict):
                os.makedirs(local_file_dict)

            local_file = '%s/%s.%s.mp3' % (local_file_dict, track['id'], track['name'])
            if not os.path.isfile(local_file):
                print('downloading: ' + track['name'])
                # res = requests.get(file_url, proxies=random_proxies(), headers=headers)
                res = requests.get(file_url, headers=headers)
                with open(local_file, 'wb') as f:
                    f.write(res.content)
                    f.close()
                print('done.')
            else:
                print('break: ' + track['name'])


if __name__ == '__main__':
    spider_queue = queue.Queue()

    luoo = LuooSpider(luoo_site, vols=['680', '721', '725', '720'], queue=spider_queue)
    # luoo.setDaemon(True) # 这个有问题
    luoo.setDaemon(False)
    luoo.start()
    # luoo.run() # 测试用

    # 测试下载 begin
    # luoo_download = LuooDownloader(luoo_site_mp3, 'D:/luoo', queue=spider_queue)
    # luoo_download.setDaemon(False)
    # luoo_download.run()
    # 测试下载 end

    # 多线程下载
    downloader_count = 5
    for i in range(downloader_count):
        luoo_download = LuooDownloader(luoo_site_mp3, 'D:/luoo', queue=spider_queue)
        luoo_download.setDaemon(False)
        luoo_download.start()

4一点感悟

学了这点之后，发现Python作为脚本语言，第三方插件真多。

这也许就是他的优势！