以下是从163源下载openstack-ocata版软件包的脚本：
from html.parser import HTMLParser
from urllib import request
import urllib
import os,sys
import socket
class myparser(HTMLParser):
'''找到a标签并把属性的值放到列表里'''
def __init__(self):
HTMLParser.__init__(self)
self.links = []
def handle_starttag(self, tag, attrs):
if tag == 'a':
if len(attrs) == 0:
pass
else:
for (variable,value) in attrs:
# print(value)
if variable == 'href':
self.links.append(value)
def callbackfunc(blocknum, blocksize, totalsize):
'''回调函数,打印下载进度
@blocknum: 已经下载的数据块
@blocksize: 数据块的大小
@totalsize: 远程文件的大小
'''
percent = int(100.0 * blocknum * blocksize / totalsize)
if totalsize > 505528:
pass
else:
percent = 100
sys.stdout.write('\r')
sys.stdout.write(file_name + percent * '>' + str(percent) + '%')
sys.stdout.flush()
def create_dir(root_tree,catalog):
'''根据url的目录结构在本地穿件文件夹'''
os.chdir(root_tree)
try:
os.makedirs(catalog)
except FileExistsError as e:
pass
def download_file(url,down_path):
'''下载文件保存到相应的目录，并把下载失败的放在一个字典里'''
global file_name
global error_download
file_name = url.split('/')[-1]
error_download = {}
socket.setdefaulttimeout(30)
try:
request.urlretrieve(url,down_path,callbackfunc)
except socket.gaierror as e:
error_download[url] = down_path
print('socket.gaierror' , url)
except urllib.error.URLError as e:
error_download[url] = down_path
print('urllib.error.URLError',url)
sys.stdout.write('\n')
def get_url_tree(url_tree):
'''获取一个字典，链接：目录，并把文件夹创建及把文件下载'''
url_tree_dict = {}
level = 0
for url in url_tree:
response = request.urlopen(url)
page = response.read().decode('utf-8')
hp = myparser()
hp.feed(page)
hp.close()
try:
hp.links.remove("../")
except ValueError as e:
pass
for file in hp.links:
if '/' in file:
create_dir(url_tree[url], file)
url_tree_dict[url+file] = url_tree[url]+file
else:
download_file(url+file,url_tree[url]+file)
if file.find('/') > 0:
level += 1
return url_tree_dict , level
url_tree = {"http://mirrors.163.com/centos/7/cloud/x86_64/openstack-ocata/":'/centos/7/cloud/x86_64/openstack-ocata/'}
try:
os.makedirs('/centos/7/cloud/x86_64/openstack-ocata/')
except FileExistsError as e:
pass
while True:
url_tree,level = get_url_tree(url_tree)
if level == 0:
break
print(url_tree,level)
for key in error_download:
download_file(key,error_download[key])
