requests+BeautifulSoup+theading实现生产者消费者模式简单爬虫处理男人装图片

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/7/11 18:57
# @Author  : 李振华

# 多线程快速抓取某个网页全部图片

import lxml
import threading
import requests
import queue
from bs4 import BeautifulSoup
from random import Random
import os


# 解析图片地址
def parse(url):
    html = requests.get(url).text
    soup = BeautifulSoup(html, 'lxml')
    for img_url in soup.find_all('img'):
        q.put(img_url.get('src'))
    print(q.qsize())


# 生成随机数
def random_str(randomlength=8):
    strs = ''
    chars = 'AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789'
    length = len(chars) - 1
    random = Random()
    for i in range(randomlength):
        strs += chars[random.randint(0, length)]
    return strs


# 下载图片
def download_image(image_url):
    if not os.path.exists('image'):
        os.mkdir('image')
    image_name = random_str()
    image = requests.get(image_url)
    with open('image/%s.jpg' % image_name, 'wb') as img:
        for b in image.iter_content(2048):
            img.write(b)


# 主函数
if __name__ == '__main__':
    q = queue.Queue()
    url = 'http://enrz.com/fhm/2017/05/11/90122.html'
    parse_thread = threading.Thread(target=parse, args=(url, ))
    parse_thread.start()
    parse_thread.join()
    for i in range(q.qsize()):
        download_image(q.get())

    download_thread_pool = []
    for i in range(q.qsize()):
        download_thread = threading.Thread(target=download_image, args=(q.get(), ))
        download_thread.start()
        download_thread_pool.append(download_thread)

    for thread in download_thread_pool:
        thread.join()