Python爬上不得姐 并将段子写入数据库
#Python2.7 可以优化一下 前10页 每页点赞最多的段子 百思不得姐
# -*- coding: utf-8 -*-
import MySQLdb
import urllib,urllib2
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
class TestBDJ():
def __init__(self):
pass
def getbdj(self):
lis = []
for h in range(1,10):
cc = []
tt = []
try:
url = 'http://www.budejie.com/text/'+str(h)
res = urllib.urlopen(url).read()
#print res
m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i> <span>(.*?)</span>',re.S) #当前版本可以这样匹配 没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1]) #把每个段子点赞的数量放入list
aa = cc
aa = map(eval,aa) #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1): #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5): #取每页前5条段子
duanzi = []
index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
#print tt[index][0].replace("<br />","")
#print tt[index][1]
star = tt[index][1] #点赞人数
word = tt[index][0].replace("<br />","").encode('utf-8') #文字内容
duanzi.append(star)
duanzi.append(word)
lis.append(duanzi)
except Exception,e:
print e
return lis
def load(self):
key = self.getbdj()
if key:
try:
conn = MySQLdb.connect(host='localhost', user='hehehe', passwd='****', db='tester', port=330,charset="utf8") # 加上charset="utf8" 指定编码格式 解决写入mysql中文乱码的问题
cur = conn.cursor()
conn.select_db('tester')
cur.execute('create table if not exists budejie(st int,info TEXT)')
for k in key:
cur.execute('insert into budejie values(%s,%s)',k)
cur.close()
conn.commit()
conn.close()
except MySQLdb.Error, e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
return 0
if __name__=='__main__':
h = TestBDJ()
h.load()
Python 3 的需要稍微修改一下
#coding:utf-8 import urllib.request import re import importlib,sys for h in range(1,10): cc = [] tt = [] try: url = 'http://www.budejie.com/text/'+str(h) res = urllib.request.urlopen(url).read() res = res.decode("utf-8") # print (res) m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i> <span>(.*?)</span>',re.S) #当前版本可以这样匹配 没有匹配作者 tt = re.findall(m,res) for i in tt: cc.append(i[1]) #把每个段子点赞的数量放入list aa = cc aa = list(map(eval,aa)) #需要将点赞的值由string转换成int #print "befor %s",aa for b in range(len(aa)-1): #排序 for i in range(len(aa) - 1): if aa[i] < aa[i + 1]: aa[i], aa[i + 1] = aa[i + 1], aa[i] #print "after %s",aa for t in range(5): #取每页前5条段子 index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子 print (tt[index][0].replace("<br />","")) print (tt[index][1]) except Exception as e: print (e)
糗事百科，稍稍做了一下处理
# -*- coding: utf-8 -*- import urllib,urllib2 import sys,re reload(sys) sys.setdefaultencoding('utf-8') for h in range(1,8): cc = [] tt = [] try: url = 'http://www.qiushibaike.com/text/page/'+str(h)+'/?s=4891212' req = urllib2.Request(url) req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36') res = urllib2.urlopen(req).read() #print res m = re.compile('<div class="content">(.*?)</div>.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S) #当前版本可以这样匹配 没有匹配作者 tt = re.findall(m,res) for i in tt: cc.append(i[1]) #把每个段子点赞的数量放入list
aa = cc aa = map(eval,aa) #需要将点赞的值由string转换成int #print "befor %s",aa for b in range(len(aa) - 1): #排序 for i in range(len(aa) - 1): if aa[i] < aa[i + 1]: aa[i], aa[i + 1] = aa[i + 1], aa[i] #print "after %s",aa for t in range(5): #取每页前5条段子 index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
print tt[index][0].replace("<br/>","")
print tt[index][1]
except Exception,e: print e
爬取含图片的段子：
# -*- coding: utf-8 -*- import urllib,urllib2 import sys,re reload(sys) sys.setdefaultencoding('utf-8') for h in range(1,3): cc = [] tt = [] try: url = 'http://www.qiushibaike.com/imgrank/page/'+str(h)+'/?s=4891221' req = urllib2.Request(url) req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36') res = urllib2.urlopen(req).read() #print res m = re.compile('<div class="content">(.*?)</div>.*?target="_blank">.*?<img src=\"(.*?)\" alt=.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S) #当前版本可以这样匹配 没有匹配作者 tt = re.findall(m,res) for i in tt: cc.append(i[2]) #把每个段子点赞的数量放入list # print i[0] # print i[1] # print i[2] aa = cc aa = map(eval,aa) #需要将点赞的值由string转换成int #print "befor %s",aa for b in range(len(aa) - 1): #排序 for i in range(len(aa) - 1): if aa[i] < aa[i + 1]: aa[i], aa[i + 1] = aa[i + 1], aa[i] #print "after %s",aa for t in range(5): #取每页前5条段子 index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子 print index print tt[index][0] print tt[index][1] print tt[index][2] except Exception,e: print e
