python3----练习题(爬取电影天堂资源,大学排名,淘宝商品比价)

import requests
import re

url = 'http://www.ygdy8.net/html/gndy/dyzz/list_23_{}.html'
for n in range(1, 2):
    new_url = url.format(n)
    html_1 = requests.get(new_url)
    html_1.encoding = 'gb2312'
    detil_list = re.findall('<a href="(.*?)" class="ulink">', html_1.text)

    for m in detil_list:
        b_url = 'http://www.ygdy8.net' + m
        html_2 = requests.get(b_url)
        html_2.encoding = 'gb2312'
        ftp = re.findall('<a href="(.*?)">.*?</a></td>', html_2.text)
        with open('tddy.txt', 'a', encoding='utf-8') as f:
            f.write(ftp[0] + '\n')

 

大学排名练习

import bs4
import requests
from bs4 import BeautifulSoup

def get_html_text(url):
    try:
        r = requests.get(url, timeout=20)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return " "


def fill_univ_list(ulist, html):
    soup = BeautifulSoup(html, "html.parser")
    for tr in soup.find('tbody').children:
        if isinstance(tr, bs4.element.Tag):   # 判断类型
            tds = tr('td')
            ulist.append([tds[0].string, tds[1].string, tds[3].string])


def print_univ_list(ulist, num):
    tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
    print(tplt.format("排名", "学校名称", "总分", chr(12288)))
    for i in range(num):
        u = ulist[i]
        print(tplt.format(u[0], u[1], u[2], chr(12288)))


def main():
    uinfo = []
    url = 'http://www.zuihaodaxue.com/zuihaodaxuepaiming2016.html'
    html = get_html_text(url)
    fill_univ_list(uinfo, html)
    print_univ_list(uinfo, 20)


main()

 淘宝商品比价:

import requests
import re

def get_html_text(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ""


def parse_page(ilt, html):
    try:
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"', html)
        tlt = re.findall(r'\"raw_title\"\:\".*?\"', html)
        for i in range(len(plt)):
            price = eval(plt[i].split(':')[1])
            title = eval(tlt[i].split(':')[1])
            ilt.append([price, title])
    except:
        print("")

def print_goods_list(ilt):
    tplt = "{:4}\t{:8}\t{:16}"
    print(tplt.format("序号", "价格", "商品名称"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count, g[0], g[1]))

def main():
    goods = '减肥餐'
    depth = 2
    start_url = 'http://s.taobao.com/search?q=' + goods
    info_list = []
    for i in range(depth):
        try:
            url = start_url + '&s=' + str(44*i)
            html = get_html_text(url)
            parse_page(info_list, html)
        except:
            continue
    print_goods_list(info_list)

 股票数据:

import re
import traceback

import requests
import sys
from bs4 import BeautifulSoup


def get_html_text(url, code='utf-8'):
    headers ={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}
    try:
        r = requests.get(url, timeout=20, headers=headers)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return ""

def get_stock_list(lst, stock_url):
    html = get_html_text(stock_url, 'GB2312')
    soup = BeautifulSoup(html, 'html.parser')
    a = soup.find_all('a')
    for i in a:
        stock_code = re.findall(r'[s][hz]\d{6}', str(i))
        if len(stock_code) != 0:
            lst.append(stock_code)


def get_stock_info(lst, stock_url, fpath):
    count = 0
    for stock in lst:
        url = stock_url + stock[0] + '.html'
        print(url)
        html = get_html_text(url)
        try:
            if html == "":
                continue
            info_dict = {}
            soup = BeautifulSoup(html, 'html.parser')
            stock_info = soup.find('div', attrs={'class': 'stock-bets'})
            info_dict.update({'股票名称': stock_info.text.split()[0]})

            key_list = stock_info.find_all('dt')
            value_list = stock_info.find_all('dd')
            for i in range(len(key_list)):
                key = key_list[i].text
                info_dict[key] = value_list[i].text

            with open(fpath, 'a', encoding='utf-8') as f:
                f.write(str(info_dict) + '\n')
                count = count + 1
                print("\r当前进度: {:.2f}%".format(count*100/len(lst), end=""))
        except:
            traceback.print_exc(file=sys.stdout)
            count = count + 1
            print("\r当前进度: {:.2f}%".format(count * 100 / len(lst), end=""))
            continue

def main():
    stock_list_url = 'http://quote.eastmoney.com/stocklist.html'
    stock_info_url = 'http://gupiao.baidu.com/stock/'
    output_file = 'D:/BaiduStockInfo.txt'
    slist = []
    get_stock_list(slist, stock_list_url)
    get_stock_info(slist, stock_info_url, output_file)