毛概复习资料spider

为了复习，把毛概题库趴下来。

github:https://github.com/xzb123/-spider

# encoding = 'utf-8'
import requests,re,os
from bs4 import BeautifulSoup

#name 表示文件名字
#url 表示题目路径
#将这个路径的题目保存为name文件
def save_html(name,url):
    '''
    #抓取页面
    res = requests.get(url,headers = headers)
    res.encoding = 'utf-8-bom'
    #页面内容
    text = res.content
    #内容写入一个文件
    f = open("temp.html",'wb')
    f.write(text)
    f.close()
    '''
    get_html("temp.html",url)
    #读出内容
    tf = open("temp.html",'r')
    input = tf.read()
    tf.close()
    
    #正则表达式替换掉部分内容
    output1 = re.sub('<iframe (.*)',' ',input)
    output2 = re.sub("<input type='hidden' (.)* value",' ',output1)
    #将最终内容写入name文件
    f = open(name,'w')
    f.write(output2)
    f.close()
    print(name," has saved ok!")

#根据mom.html文件内容得到需要进一步爬取的sonurl
def get_sonurl(url_list):
    f = open('mom.html','r')
    text = f.read()
    soup = BeautifulSoup(text,'html.parser')
    f.close()
    a_list = soup.find_all('a')
    for i in a_list:
#        print(i['href'])
        #find_all得到的list中每个是slice对象，可以直接得到属性，很舒服呀~
        url_list.append(i['href'])
#    print(url_list)
    print("find sonurl ok!")

#根据url获取mom.html
def get_html(name,url):
    f = open(name,'wb')
    res = requests.get(url,headers = headers)
    res.encoding = 'utf-8-bom'
    text = res.content
    f.write(text)
    f.close()
    print(name + " " + url," has get ok!")
    
if __name__ == "__main__":
    print("welcome!")
    print("input your cookie:\nlike this \nJSESSIONID=7CBC32078B041854FE8EFAEE1A961E98; DWRSESSIONID=Eghu$3ODyHz35BCqw4xEZV5ugJm")
    cookie = 'SESSIONID=7CBC32078B041854FE8EFAEE1A961E98; DWRSESSIONID=Eghu$3ODyHz35BCqw4xEZV5ugJm'
    cookie = input()
    print("input path:\nlike this \n xzb/")
    path = input()
    if os.path.exists(path):
        pass
    else:
        os.mkdir(path)
    #header信息，主要是cookie
    headers = {}
    headers['Cookie'] = cookie
    
    momurl = 'http://222.195.158.247/meol/common/question/test/student/stu_qtest_more_result.jsp?testId=22400165'
    faurl = 'http://222.195.158.247/meol/common/question/test/student/'
    #得到mom.html
    get_html("mom.html",momurl)
    url_list = []
    #解析mom.html
    get_sonurl(url_list)
#    print(url_list)
    cnt = 1
    for sonurl in url_list:
        url = faurl + sonurl
        name = path + str(cnt) + '.html'
        print(name)
        cnt = cnt+1
        save_html(name,url)
    os.remove("temp.html")
    os.remove("mom.html")
    
    print("finish!")