Python.51job

"""
https://www.51job.com/
"""

import requests
import requests.adapters
import string
import urllib.parse
from lxml import etree
import redis
from pyquery import PyQuery as pq
import multiprocessing
import pymongo
import datetime
import xlsxwriter

retries = 5  # 最大尝试次数
redis_key_page = 'redis_key_page'
redis_key_detail = 'redis_key_detail'


def get_url_txt(url, headers, encoding, data=None):
    ret = ''
    try:
        requests.adapters.DEFAULT_RETRIES = 5
        session = requests.session()
        session.keep_alive = False
        if data is None:
            response = session.get(url, headers=headers)
        else:
            response = session.get(url, headers=headers, data=data)
        if response.status_code == 200:
            response.encoding = encoding
            ret = response.text
        response.close()
        session.close()
    except Exception as e:
        print(e)
    return ret


def encode_url(keys):
    try:
        keys_str = urllib.parse.quote(keys, safe=string.printable)
        keys_str_ = ''
        for i in keys_str.split('%'):
            if len(i):
                keys_str_ += ('%25' + i)
        return keys_str_
    except Exception as e:
        print(e)
    return ''


def get_page_count(keys):
    try:
        keys_str = encode_url(keys)
        url1 = 'https://search.51job.com/list/030000,000000,0000,00,9,99,'
        url2 = keys_str
        url3 = ',2,'
        url4 = str(1)
        url5 = '.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
        url = url1 + url2 + url3 + url4 + url5
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
        encoding = 'gbk'
        text = get_url_txt(url=url, headers=headers, encoding=encoding)
        # print(text)
        xpath = etree.HTML(text)
        page_count = ''.join(xpath.xpath('//*[@id="resultList"]/div[2]/div[5]//text()'))
        page_count = int(page_count.split('/')[1].replace(' ', ''))
        # print(page_count)
        return page_count
    except Exception as e:
        print(e)
    return 0


def flush_page_href():
    try:
        redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)).delete(
            redis_key_page)
    except Exception as e:
        print(e)


def get_page_href(keys):
    try:
        flush_page_href()
        page_count = get_page_count(keys)
        keys_str = encode_url(keys)
        url1 = 'https://search.51job.com/list/030000,000000,0000,00,9,99,'
        url2 = keys_str
        url3 = ',2,'
        url5 = '.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
        for i in range(page_count):
            url4 = str(1 + i)
            url = url1 + url2 + url3 + url4 + url5
            redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)).rpush(
                redis_key_page, '0|' + url)
    except Exception as e:
        print(e)


def flush_detail_href():
    try:
        redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=1)).delete(
            redis_key_detail)
    except Exception as e:
        print(e)


def get_detail_href_(redis_value):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
        encoding = 'gbk'
        conn_count = int(redis_value.split('|')[0])
        page_url = redis_value.split('|')[1]
        text = get_url_txt(url=page_url, headers=headers, encoding=encoding)
        if len(text):
            doc1 = pq(text)
            doc2 = doc1('.t1')
            # print(doc2)
            for i in doc2:
                title = pq(i).find('a').attr('title')
                href = pq(i).find('a').attr('href')
                if title is None and href is None:
                    continue
                # print(title, href)
                redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=1)).rpush(
                    redis_key_detail, '0|' + href)
        elif conn_count < retries:
            redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)).rpush(
                redis_key_page, str(conn_count + 1) + '|' + page_url)
    except Exception as e:
        print(e)


def get_detail_href():
    try:
        flush_detail_href()
        while redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)).llen(
                redis_key_page) > 0:
            p = multiprocessing.Pool()
            while True:
                redis_value = redis.StrictRedis(
                    connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=0)).lpop(
                    redis_key_page)
                if redis_value is None:
                    break
                redis_value = redis_value.decode(encoding='utf8', errors='ignore')
                # print(redis_value)
                p.apply_async(get_detail_href_, (redis_value,))
            p.close()
            p.join()
    except Exception as e:
        print(e)


def get_detail_info_(redis_value):
    company, position, salary, require, contack, introduce = '', '', '', '', '', ''
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
        encoding = 'gbk'
        conn_count = int(redis_value.split('|')[0])
        detail_href = redis_value.split('|')[1]
        text = get_url_txt(url=detail_href, headers=headers, encoding=encoding)
        if len(text):
            xpath = etree.HTML(text)
            company = xpath.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/p[1]/a[1]//text()')
            position = xpath.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/h1//text()')
            salary = xpath.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/strong//text()')
            require = xpath.xpath('/html/body/div[3]/div[2]/div[3]/div[1]//text()')
            contack = xpath.xpath('/html/body/div[3]/div[2]/div[3]/div[2]//text()')
            introduce = xpath.xpath('/html/body/div[3]/div[2]/div[3]/div[3]//text()')
            company = ''.join(company).strip()
            position = ''.join(position).strip()
            salary = ''.join(salary).strip()
            require = ''.join(require).replace('\r\n', '').strip()
            contack = ''.join(contack).replace('\r\n', '').strip()
            introduce = ''.join(introduce).replace('\r\n', '').strip()
            # print(company, position, salary, require, contack, introduce)
            pymongo.MongoClient('localhost:27017')['db']['table'].insert_one(
                {'company': company,
                 'position': position,
                 'salary': salary,
                 'require': require,
                 'contack': contack,
                 'introduce': introduce, })
        elif conn_count < retries:
            redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=1)).rpush(
                redis_key_detail, str(conn_count + 1) + '|' + detail_href)
    except Exception as e:
        print(e)
    return company, position, salary, require, contack, introduce


def get_detail_info():
    try:
        while redis.StrictRedis(connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=1)).llen(
                redis_key_detail) > 0:
            p = multiprocessing.Pool()
            while True:
                redis_value = redis.StrictRedis(
                    connection_pool=redis.ConnectionPool(host='127.0.0.1', port=6379, db=1)).lpop(
                    redis_key_detail)
                if redis_value is None:
                    break
                redis_value = redis_value.decode(encoding='utf8', errors='ignore')
                # print(redis_value)
                p.apply_async(get_detail_info_, (redis_value,))
            p.close()
            p.join()
    except Exception as e:
        print(e)


if __name__ == '__main__':
    pass
    # headers = {
    #     'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
    # encoding = 'gbk'
    # print(get_url_txt('https://jobs.51job.com/guangzhou-thq/114903412.html?s=01&t=0', headers, encoding))
    pass
    # print(encode_url('逆向'))
    # print(get_page_count('逆向'))
    # get_page_href('逆向')
    # get_detail_href_(
    #     'https://search.51job.com/list/030000,000000,0000,00,9,99,%25E9%2580%2586%25E5%2590%2591,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=')
    # get_detail_href()
    # print(get_detail_info_('https://jobs.51job.com/guangzhou-thq/119363494.html?s=01&t=0'))
    # get_detail_info()
    # pymongo.MongoClient('localhost:27017')['db']['table'].insert_one({'k1': 'v1', 'k2': 'v2'})
    pass
    # start = datetime.datetime.now()
    # print(start.strftime('%Y-%m-%d %H:%M:%S'))
    # pymongo.MongoClient('localhost:27017')['db']['table'].drop()
    # get_page_href('逆向')
    # get_detail_href()
    # get_detail_info()
    # end = datetime.datetime.now()
    # print(end.strftime('%Y-%m-%d %H:%M:%S'))
    # print('cost seconds : %d' % (end - start).seconds)
    pass
    cols = len(pymongo.MongoClient('localhost:27017')['db']['table'].find_one())
    book = xlsxwriter.Workbook('query.xlsx')
    sheet = book.add_worksheet('sheet1')
    row = 0
    for i in pymongo.MongoClient('localhost:27017')['db']['table'].find({}):
        sheet.write(row, 0, i['company'])
        sheet.write(row, 1, i['position'])
        sheet.write(row, 2, i['salary'])
        sheet.write(row, 3, i['require'])
        sheet.write(row, 4, i['contack'])
        sheet.write(row, 5, i['introduce'])
        row += 1
    book.close()
    pass