世纪佳缘信息爬取存储到mysql,下载图片到本地，从数据库选取账号对其发送消息更新发信状态
利用这种方法，可以把所有会员信息存储下来，多线程发信息，10秒钟就可以对几百个会员完成发信了。
首先是筛选信息后爬取账号信息，
#-*-coding:utf-8-*- import requests,re,json,time,threadpool,os from mydba import MySql from gevent import monkey #monkey.patch_all() header={ 'Cookie':'guider_quick_search=on; SESSION_HASH=xxxxxxxxcd4523713c3d3700579350daa9992f1a; user_access=1; save_jy_login_name=1314xxxx; sl_jumper=%26cou%3D17; last_login_time=1498576743; user_attr=000000; pclog=%7B%22160961843%22%3A%221498576757268%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A8677484%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%220%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22m%22%3A26%2C%22f%22%3A0%2C%22omc%22%3A0%7D; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjnX7Fu00000PNeYH300000uybcI1.THL2sQ1PEPZRVfK85yF9pywd0ZnqryRkryfdryDsnj0kuWR3u0Kd5RuKP17Knj97rDDvPRNAPH9APbRkwHc4njIAnDPjP1NA0ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqnWRdnjn&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; REG_REF_URL=http://www.jiayuan.com/usercp/profile.php?action=work; PHPSESSID=d5557cc5a4560c14c2dc685bcb7fc009; stadate1=159xxxx; myloc=44%7C4403; myage=xx; PROFILE=160xxxxxx%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=159xxxxxxx; myincome=40; mylevel=2; main_search:160961843=%7C%7C%7C00; RAW_HASH=J3ewrCGVZG5eU2agrymP2-bNz0IQQpivXOncdsOO63oS3%2AH4o4%2AsAifV0twuNFXqWippm3rMHXvTK%2APmGbap-ZYZpz-18ogWLnBXkcpY87GlFps.; COMMON_HASH=424558c12fxxxxx40ba148bcac4fb; IM_CON=%7B%22IM_TM%22%3A1498577459991%2C%22IM_SN%22%3A3%7D; IM_M=%5B%5D; pop_time=1498577573654; IM_CS=2; IM_ID=12; is_searchv2=1; IM_TK=1498577921417', 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', } def fun(i): url='http://search.jiayuan.com/v2/search_v2.php?sex=f&key=&stc=1%3A4403%2C2%3A23.27%2C3%3A155.170%2C23%3A1&sn=default&sv=1&p='+str(i)+'&f=select&listStyle=bigPhoto&pri_uid=xxxx&jsversion=v5' while(1): res='' try: res=requests.get(url,headers=header) except Exception,e: print e if res!='': break content=res.content dictx= json.loads(re.findall('##jiayser##([\s\S]*?)##jiayser##',content)[0]) ##content是一个json但加了其他东西，需要把json提取出来 #print dictx for dcx in dictx['userInfo']: #print dcx uhash=dcx['helloUrl'].encode('utf8').split('uhash=')[-1] crawled_time=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #print uhash sql='''insert into shijijiayuan values(NULL,"%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'''%(dcx['nickname'],dcx['age'],dcx['sex'],dcx['education'],dcx['height'],uhash,dcx['image'],dcx['marriage'],dcx['matchCondition'],\ dcx['uid'],dcx['work_location'],dcx['work_sublocation'],dcx['shortnote'],crawled_time) #print '要插入的sql语句是：',sql if not os.path.exists('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid'])): pass while(1): res2='' try: res2=requests.get(dcx['image'],headers=header,timeout=30) except Exception,e: print e if res2!='': break try: with open('images/%s_%s.jpg'%(dcx['nickname'],dcx['uid']),'wb') as f: f.write(res2.content) print '%s保存图片成功%s'%(dcx['uid']) except Exception,e: print '%s保存图片失败%s'%(dcx['uid'],str(e)) sql2 = 'select 1 from shijijiayuan where uid=%s'%dcx['uid'] mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8') mysqlx2.query(sql2) is_exist = mysqlx2.cursor.fetchone() ##判断下数据库有没有这个uid，uid是账号的唯一 if is_exist==(1,): print '已存在%s'%dcx['uid'] else: mysqlx2.query(sql) del mysqlx2 pool = threadpool.ThreadPool(300) ##用thredpool线程池开300线程 requestsx = threadpool.makeRequests(fun,[i for i in range(1,500)]) [pool.putRequest(req) for req in requestsx] pool.wait()
执行这个后，就可以看到数据库里面会有很多会员信息了。
附上建表语句
CREATE TABLE `shijijiayuan` (
`id` int(10) NOT NULL AUTO_INCREMENT,
`nickname` varchar(20) DEFAULT NULL,
`age` int(5) DEFAULT NULL,
`sex` varchar(5) DEFAULT NULL,
`education` varchar(30) DEFAULT NULL,
`height` int(5) DEFAULT NULL,
`uhash` varchar(50) DEFAULT NULL,
`image` varchar(160) DEFAULT NULL,
`marriage` varchar(10) DEFAULT NULL,
`matchCondition` varchar(200) DEFAULT NULL,
`uid` int(15) DEFAULT NULL,
`work_location` varchar(10) DEFAULT NULL,
`work_sublocation` varchar(10) DEFAULT NULL,
`shortnote` varchar(1000) DEFAULT NULL,
`crawled_time` datetime DEFAULT NULL,
`last_hello_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`hello_times` int(3) DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8
这个是下载的图片。
第二部分是对会员发送信息。
#-*-coding:utf-8-*- import requests,re,json,time,threadpool,os,random from concurrent.futures import ThreadPoolExecutor from mydba import MySql from gevent import monkey from greetings import greeting_list header={ 'Cookie':'guider_quick_search=on; SESSION_HASH=yyyyyyya6f4b944ea522fb669bd758f17ea00; FROM_BD_WD=%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598; FROM_ST_ID=416640; FROM_ST=.jiayuan.com; REG_ST_ID=15; REG_ST_URL=http://bzclk.baidu.com/adrc.php?t=06KL00c00f7t0wC0Gfum0QkHAsjb97wu00000PNeYH300000v9ZrmW.THL2sQ1PEPZRVfK85yF9pywd0Znqrj6vPWm1nH0snj0sm1N9PsKd5HTYwHTvrRmLfH9anDmLwHFjfRwArj6sfRcLPWTkn1610ADqI1YhUyPGujYzrH0dnWTYnHckFMKzUvwGujYkP6K-5y9YIZ0lQzqzuyT8ph-9XgN9UB4WUvYETLfE5v-b5HfkPWmYnaudThsqpZwYTjCEQLILIz4Jpy74Iy78QhPEUfKWThnqPWfdns&tpl=tpl_10762_15668_1&l=1053916117&attach=location%3D%26linkName%3D%25E6%25A0%2587%25E9%25A2%2598%26linkText%3D%25E4%25B8%2596%25E7%25BA%25AA%25E4%25BD%25B3%25E7%25BC%2598%25E7%25BD%2591%25EF%25BC%2588Jiayuan.com%25EF%25BC%2589%253A2017%25EF%25BC%258C%26xp%3Did(%2522m78828180%2522)%252FDIV%255B1%255D%252FDIV%255B1%255D%252FDIV%255B1%255D%252FH2%255B1%255D%252FA%255B1%255D%26linkType%3D%26checksum%3D191&ie=UTF-8&f=8&tn=baidu&wd=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&oq=%E4%B8%96%E7%BA%AA%E4%BD%B3%E7%BC%98&rqlang=cn; user_access=1; _gscu_1380850711=98711257z5x28w21; _gscbrs_1380850711=1; save_jy_login_name=xxxxx; last_login_time=1498711280; user_attr=000000; pclog=%7B%22160961843%22%3A%221498711275466%7C1%7C0%22%7D; IM_S=%7B%22IM_CID%22%3A7816174%2C%22IM_SV%22%3A%22123.59.161.3%22%2C%22svc%22%3A%7B%22code%22%3A0%2C%22nps%22%3A0%2C%22unread_count%22%3A%221%22%2C%22ocu%22%3A0%2C%22ppc%22%3A0%2C%22jpc%22%3A0%2C%22regt%22%3A%221486465367%22%2C%22using%22%3A%2240%2C33%2C2%2C%22%2C%22user_type%22%3A%2210%22%2C%22uid%22%3A160961843%7D%2C%22m%22%3A12%2C%22f%22%3A0%2C%22omc%22%3A0%7D; REG_REF_URL=http://www.jiayuan.com/msg/?from=menu; PHPSESSID=8902adb4512b9e8904156c8b4267dc4c; stadate1=159961843; myloc=44%7C4403; myage=xx; PROFILE=160961843%3A%25E9%25A3%258E%3Am%3Aat1.jyimg.com%2F42%2Ffb%2F4558c12f370e446240ba148bcac4%3A1%3A%3A1%3A4558c12f3_2_avatar_p.jpg%3A1%3A1%3A61%3A10; mysex=m; myuid=15xxxx; myincome=40; mylevel=2; main_search:160xxxx=%7C%7C%7C00; RAW_HASH=suldoq0mMteEqQAs3qWKnMx-9sll5N8EDN-tu7bwGBA%2AtITMUwUbnL59YPR8ydVDXCxjt19iNPlu8kzi-jp8cU%2AzbhYS1iuOJyt%2APg32BhCebEE.; COMMON_HASH=424xxxx446240ba148bcac4fb; IM_CS=2; IM_ID=3; is_searchv2=1; pop_time=1498713117447; IM_CON=%7B%22IM_TM%22%3A1498713128148%2C%22IM_SN%22%3A9%7D; IM_M=%5B%5D; IM_TK=1498713324458', 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36', } mysqlx = MySql('localhost', 'root', '123456', 'test', 'utf8') mysqlx.query("SELECT nickname,uhash FROM shijijiayuan order by id desc limit 10,5") #SELECT * FROM shijijiayuan where hello_times=0 limit 1,1 #select语句选取几个账号发信 fetchallx= mysqlx.cursor.fetchall() nickname_uhash_list=[ft for ft in fetchallx] url2='http://www.jiayuan.com/msg/dosend.php?type=hello&randomfrom=4' url3='http://www.jiayuan.com/165508547' print nickname_uhash_list def hello(nickname,uhash): datax2={ 'textfield':'嗨。%s妹子。'%(nickname)+random.choice(greeting_list), ##问候语是一个列表，随机发送一个信息给他。 'list':'', 'category':'2', 'xitong_zidingyi_wenhouyu':'1', 'xitong_zidingyi_wenhouyu':'0', 'new_type':'1', 'pre_url':'', 'sendtype':'20', 'hellotype':'hello', 'pro_id':'0', 'new_profile':'3', 'to_hash':uhash, #9c248d2c96868695640e76ed74e9d3b0 f11e574d44c7967b88afb9940392c14b uhash的值决定对谁发送信息，不是uid来决定，接口中没有uid参数 'fxly':'search_v2', 'tj_wz':'none', 'need_fxtyp_tanchu':'0', 'self_pay':'0', 'fxbc':'0', 'cai_xin':'0', 'zhuanti':'0', 'liwu_nofree':'0', 'liwu_nofree_id':'88', } try: contentx='空' resx= requests.post(url2,data=datax2,headers=header) contentx= resx.content print contentx if '发送成功' in contentx: ##如果发送信息成功，就把数据库中的最后发信时间更新下，然后把发信次数加1，以便于之后用slect语句选定没有发信的人对其发信 mysqlx2 = MySql('localhost', 'root', '123456', 'test', 'utf8') #update student set score=score+1 where id = 1 sql='''UPDATE shijijiayuan SET hello_times=hello_times+1 WHERE uhash="%s"'''%(uhash) #print sql mysqlx2.query(sql) del mysqlx2 except Exception,e: print e poolx = ThreadPoolExecutor(max_workers=10) ##这也是多线程发信，世纪佳缘网站发信接口，响应时间非常巨大，浪费时间，用多线程Threadpoolexurot非常好的包，是未来要做的事，把一切东西扔到这里面，本篇没有体现出比threadpool好的优点。 [poolx.submit(hello,i[0],i[1]) for i in nickname_uhash_list]
执行完这个脚本后，在网页或者app中就可以看到发的信息了。
这个是打了招呼之后表的变化。
没有什么难度，就是要分析，首先是分析账号信息在哪里，是ajax，返回的是json加了一些字符，把json提取出来，转成字典，比较好弄。
然后就是要分析发信接口是哪一个url，这个根据规律猜测下然后查看下post的值就能找到，然后就是找出接口中的关键字段，关键字段不多，uhash是决定发送给谁（尝试几下就可以），textfield是发送的内容。
本文的登录是headers中的cookie字段携带。
反对极端面向过程编程思维方式，喜欢面向对象和设计模式的解读，喜欢对比极端面向过程编程和oop编程消耗代码代码行数的区别和原因。致力于使用oop和36种设计模式写出最高可复用的框架级代码和使用最少的代码行数完成任务，致力于使用oop和设计模式来使部分代码减少90%行，使绝大部分py文件最低减少50%-80%行的写法。
