爬虫

pyMqSQL类封装

import requests
from bs4 import BeautifulSoup 
import pymysql

class MySQL:
    """
    对pymysql的简单封装
    """
    def __init__(self,host,user,pwd,db):
        self.host = host
        self.user = user
        self.pwd = pwd
        self.db = db
        try:
            self.conn = pymysql.connect(host=self.host,user=self.user,password=self.pwd,database=self.db,charset="utf8")
            self.cur = self.conn.cursor()
        except pymysql.Error as e:  
            print("Mysql Error %d: %s" % (e.args[0], e.args[1])) 
    def select(self, db):
      try:  
          self.conn.select_db(db)  
      except pymysql.Error as e:  
          print("Mysql Error %d: %s" % (e.args[0], e.args[1]))  
          
    def __query(self,sql):  
        try:  
           n=self.cur.execute(sql)  
           return n  
        except pymysql.Error as e:  
           print("Mysql Error:%s\nSQL:%s" %(e,sql))
    def queryRow(self,sql):  
        self.__query(sql)  
        result = self.cur.fetchone()  
        return result  
  
    def queryAll(self,sql):  
        self.__query(sql)  
        result=self.cur.fetchall()  
        desc =self.cur.description  
        d = []  
        for inv in result:  
             _d = {}  
             for i in range(0,len(inv)):  
                 _d[desc[i][0]] = str(inv[i])  
             d.append(_d)  
        return d
  
    def insert(self,p_table_name,p_data):  
        for key in p_data:  
            p_data[key] = "'"+str(p_data[key])+"'"  
        key   = ','.join(p_data.keys())  
        value = ','.join(p_data.values())  
        real_sql = "INSERT INTO " + p_table_name + " (" + key + ") VALUES (" + value + ")"  
        print(real_sql)
        #self.query("set names 'utf8'")  
        return self.__query(real_sql)  
           
    def getLastInsertId(self):  
        return self.cur.lastrowid  
  
    def rowcount(self):  
        return self.cur.rowcount  
  
    def commit(self):  
        self.conn.commit()  
  
    def close(self):  
        self.cur.close()  
        self.conn.close()  

mysql = MySQL(host="host",user="user",pwd="pwd",db="db")
#result = mysql.insert('tb1', {'name':'zhangsan','content':'test'})
result = mysql.queryAll("SELECT * FROM tb1 ")
result = mysql.queryRow("SELECT * FROM tb1 ")
print(result)
mysql.commit()
mysql.close()

获取cookie

from selenium import webdriver
import time
import json

def get_cookie():
    chromedriver_path = r'G:\EX\Python\spider\chromedriver.exe'
    browser = webdriver.Chrome(executable_path = chromedriver_path)
    browser.get("https://passport.weibo.cn/signin/login")  
    time.sleep(2) 
    browser.find_element_by_id("loginName").send_keys('name@qq.com')  
    browser.find_element_by_id("loginPassword").send_keys('keys')  #密码   
    browser.find_element_by_id("loginAction").click() 
    time.sleep(4)
    cookie = {}
    for it in browser.get_cookies():
        cookie[it['name']] = it['value']
    json.dump(cookie, open(r'./cooike.json', 'w', encoding='utf-8'))  
    browser.close()
    return cookie