github项目

import requests
from bs4 import BeautifulSoup
url = 'https://github.com/login'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
           'Referer': 'https://github.com/',
           'Upgrade-Insecure-Requests': '1',  # 此处的1 必须是字符串，不是数字
           'Host': 'github.com',
           # 'Cookie': 'logged_in=no; _ga=GA1.2.1001043458.1529895849; _octo=GH1.1.300737173.1529895849; _gh_sess=OG5HQmlxRzQwMGpJUjdtSVRjaHdSaGlkL01qUm5TY0lqUURlUTFaN0VyV0lzelNLc25VY2FtY0ZSWFA4SnBtSStkUXNyVEI5c0FJZmt2amZHWXRwSUpQcm1PUk1BaDVicW1icXk3VjVzTlpObTJLOFNyMW5OUnRuN3FKRi9jOGI3Nm5zOFhMQnI0NEoxQ04zbHBwZjBKYlJ1eDN5R01RZndjL1dmb1MySFVkdjU4ZkpGVWUwcE1sZ0tXUHBDZmJRcXZVa2ZOK2l6bldaUnpZMDh2eEoxTXBRaCtMOG1vT2ZRZkZHMkZ6SElKK2U3VFZWQUgrNUVXeGRjQlducm5WZUNISURrclk2SmZXUG9IdWdXYVhHWGs0bG9ZWkNMbzZFNjVaM2J4bmNNM2c9LS03c2lmc0FSaHVYcUQ0UVRJQm9vczNRPT0%3D--ae466552fde9b6dd5543248d9d38a56fe2693cfe; _gat=1; tz=Asia%2FShanghai',
           'Connection': 'keep-alive',
           'Accept-Language': 'zh-CN,zh;q=0.8',
           'Accept-Encoding': 'gzip, deflate, br',
           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'}
res1 = requests.get(url,headers=headers)

soup = BeautifulSoup(res1.text, 'lxml')
tag_input = soup.find(name='input', attrs={'name': 'authenticity_token'})
authenticity_token = tag_input.get('value')
data = {'commit': 'Sign+in',
        'utf8': '✓',
        'authenticity_token': authenticity_token,
        'login': '2953889431@qq.com',
        'password': '15035942955aebcd'}

cookies=res1.cookies.get_dict()
# 这里的url是https://github.com/session，不是https://github.com/login
res2 = requests.post(url='https://github.com/session', headers=headers,cookies=cookies, data=data)
print(authenticity_token)
print(res2.status_code)
print(res2.reason)
cookies.update(res2.cookies.get_dict())
res3 = requests.get(url='https://github.com/settings/repositories',
                    cookies=cookies,
                    headers=headers
                  )

print(res3.url)
print(res3.status_code)
print(res3.reason)

soup3 = BeautifulSoup(res3.text, 'lxml')
project = soup3.find(name='div', attrs={'class': 'listgroup'})
print(project)
project_list = project.find_all(name='a', attrs={'class': 'mr-1'})
for i in project_list:
    project_name = i.text
    project_ = i.get('href')
    project_href = 'https://github.com/' + project_.split('/', maxsplit=1)[1]
    print('项目名称：%s , 项目连接：%s' % (project_name, project_href), '\n')

# 爬取github注意事项，1.以后携带的cookie使用的是登录后的cookie
                   # 2.需要在登录页面找到token,该token是动态的需要使用bs4，或者正则表达式来获取动态值