python jenkins注册脚本（zip,tgz,war之类解压缩,http请求）

#!/usr/local/python
# -*- coding: utf-8 -*-

import logging
import requests
import argparse
import sys
import os
import traceback
import json
import shutil
import tarfile
import zipfile
#################################################################
# Custom define area, modify the follow parameter when necessary
ORG =  
DEPLOY_REPO_IP = ''
DEPLOY_REPO_HOST = ''
DEPLOY_REPO_ENS = ''

DEPLOY_IP = ''
DEPLOY_HOST = ''

HEADER = {
    'org': ORG,
    'user': 'defaultUser',
    'Host': '',
}
#################################################################


# Set the logger
FORMAT = '[%(asctime)s %(filename)s(line:%(lineno)d) %(levelname)s] %(message)s'
logging.basicConfig(format=FORMAT)
logger = logging.getLogger('log')
logger.setLevel(logging.DEBUG)


# Receive a package_id by name
def receive_package_id(name,PACKAGE_TYPE):
    url = 'http://{ip}/package/search'.format(ip=DEPLOY_IP)
    params = {
        'page': 1,
        'pageSize': 500,
        'name': name,
        'type': PACKAGE_TYPE,
    }

    package_id = ''
    headers = HEADER.copy()
    headers['Host'] = DEPLOY_HOST
    resp = requests.get(url, params=params, headers=headers)
    if resp.status_code == 200:
        data = resp.json()['data']
        for package in data['list']:
            if package['name'] == name:
                package_id = package['packageId']

    return package_id
def get_config_lastversion_id(package_id):
    headers = HEADER.copy()
    headers['content-type'] = 'application/json'
    headers['Host'] = DEPLOY_HOST
    url = 'http://{DEPLOY_IP}/version/list'.format(DEPLOY_IP=DEPLOY_IP)
    params_dict = {
        "packageId":package_id,
        'page': 1,
        'pageSize': 500,
        'order':'ctime desc'
    }
    resp = requests.get(url=url, params=params_dict, headers=headers, timeout=30)
    if resp.status_code == 200:
        data=resp.json()["data"]["list"][0]
        last_config_version_id=data["versionId"]
        return last_config_version_id

def un_tar_gz(file_name, extract_path):
    """
    un.tar.gz 
    :param file_name: compressed package path 压缩包路径
    :param extract_path: decompression target path 解压路径

    :return:
    """
    if os.path.isdir(extract_path):
        pass
    else:
        logger.error('{path} not exists'.format(path=extract_path))
    tar = tarfile.open(file_name)
    names = tar.getnames()
    for name in names:
        if not str(name).endswith(".conf.yaml"):
            tar.extract(name, path=extract_path)
    tar.close()
def tar_gz(file_name,tmp_tar_dir,tmp_dir):
    """ 打包不带绝对路径
    :param file_name:   packaged name
    :param tmp_tar_dir: compressed package deposit directory 压缩包存放路径
    :param tmp_dir: packaged directory 压缩文件路径
    :return:
    """
    tar = tarfile.open(os.path.join(tmp_tar_dir,file_name),"w:gz")
    for root,dir,files in os.walk(tmp_dir):
        root_dir = os.path.relpath(root,start=tmp_dir)
        for file in files:
            full_path = os.path.join(root,file)
            tar.add(full_path,arcname=os.path.join(root_dir,file))
    tar.close()
def unpack_zipfile(filename, extract_dir):
    """Unpack zip `filename` to `extract_dir`
    """
    if not zipfile.is_zipfile(filename):
        logger.error("%s is not a zip file" % filename)
    zip = zipfile.ZipFile(filename)
    try:
        for file in zip.namelist():
            zip.extract(file, extract_dir)
    finally:
        zip.close()
def move_file(source, destination,exclude=None):
    """
    :param source:
    :param destination:
    :param exclude:
    :return:
    """
    if os.path.isfile(source):
        path, name = os.path.split(destination)
        if not os.path.exists(path):
            os.makedirs(path)
        shutil.move(source, destination)
    elif os.path.isdir(source):
        file_dir=os.listdir(source)
        if exclude:
            file_dir.remove(exclude)
        for file in os.listdir(source):
            shutil.move(source + "/" + file,destination)
    else:
        logger.error("%s not exist!" % (source,))
def downlod_file(name, file):  #download the file and repackage it
    pkg_name="dockerfile_{name}".format(name=name)
    pkg_id=receive_package_id(pkg_name,2)
    path=os.path.dirname(file)
    if not pkg_id:
        logger.error(u"can't find out the package_id of the package: {pkg_name}".format(pkg_name=pkg_name))
        sys.exit(1)
    version_id=get_config_lastversion_id(pkg_id)
    if not version_id:
        logger.error(u"can't find out the version_id of the package: {pkg_name}".format(pkg_name=pkg_name))
        sys.exit(1)
    url = 'http://{ip}/archive/{packageId}/{versionId}'.format(ip=DEPLOY_IP, packageId=pkg_id, versionId=version_id)
    headers = HEADER.copy()
    headers['Host'] = DEPLOY_REPO_HOST
    resp = requests.get(url, headers=headers, stream=True)
    if resp.status_code == 200:
        file_name = path + '/' + str(version_id) + '.tar.gz'
        with open(file_name, 'wb') as f:
            for chunk in resp.iter_content(chunk_size=512):
                if chunk:
                    f.write(chunk)
        un_tar_gz(file_name, path + "/")
        os.remove(path + "/" + str(version_id) + '.tar.gz')
        destpath = os.path.join(path, str(version_id))
        tar_file_name = "{version}.tar.gz".format(version=version_id)
        tar_file = "{file_path}.tar.gz".format(file_path=destpath)
        pkg_name = os.path.basename(file).split(".")[0]
        dest_path = "{destpath}/{name}".format(destpath=destpath, name=pkg_name)
        if os.path.basename(file).endswith(".jar"):  #determine the package type
            shutil.copy(file,destpath)
            tar_gz(tar_file_name, path, destpath)
            shutil.rmtree(destpath)
            return tar_file
        elif os.path.basename(file).endswith(".war"):
            os.system("unzip -oq {file} -d {dest}".format(file=file,dest=destpath))
            if os.path.isdir(dest_path): #determine first level directory
                move_file(destpath,dest_path,exclude=pkg_name)
                tar_gz(tar_file_name, path, dest_path)
                shutil.rmtree(destpath)
                return tar_file
            else:
                tar_gz(tar_file_name, path, destpath)
                shutil.rmtree(destpath)
                return tar_file
        elif os.path.basename(file).endswith(".tgz") or os.path.basename(file).endswith(".tar.gz"):
            un_tar_gz(file,destpath)
            if os.path.isdir(dest_path):
                move_file(destpath,dest_path,exclude=pkg_name)
                tar_gz(tar_file_name, path, dest_path)
                shutil.rmtree(destpath)
                return tar_file
            else:
                tar_gz(tar_file_name, path, destpath)
                shutil.rmtree(destpath)
                return tar_file
        elif os.path.basename(file).endswith(".zip"):
            unpack_zipfile(file,destpath)
            if os.path.isdir(dest_path):
                move_file(destpath,dest_path,exclude=pkg_name)
                tar_gz(tar_file_name, path, dest_path)
                shutil.rmtree(destpath)
                return tar_file
            else:
                tar_gz(tar_file_name, path, destpath)
                shutil.rmtree(destpath)
                return tar_file
        else:
            logger.error("unknown the compress type,please use jar,war,zip,tar.gz,tgz")
            sys,exit(1)
    else:
        logger.error(u"{0}download file failed-->>{1}".format(str(version_id) + '.tar.gz', resp.status_code))
        sys.exit(1)
# upload to remote deploy repository
def archive(pkg_name, file_path, remark, unzip, strip_first):
    package_id = receive_package_id(pkg_name,1)
    if not package_id:
        logger.error(u"{file_path}: can't find out the package_id of the package: {pkg_name}".format(
        file_path=file_path, pkg_name=pkg_name))
        sys.exit(1)

    # create the version
    if os.path.exists(file_path):
        params_dict = {
            'packageId': package_id,
            'message': remark,
            'unzip': unzip,
            'stripFirst': strip_first,
        }
        headers = HEADER.copy()
        headers['Host'] = DEPLOY_REPO_HOST
        url = 'http://{DEPLOY_REPO_IP}/archive'.format(DEPLOY_REPO_IP=DEPLOY_REPO_IP)

        try:
            fps = {'file': open(file_path, 'rb')}
            resp = requests.post(url=url, files=fps, data=params_dict, headers=headers, timeout=300)

            status_code = resp.status_code
            if status_code == 200:
                resp_format = resp.json()
                if resp_format.get('code', None) == 0:
                    data = resp_format['data']

                    logger.info(u'{file_path}: create the version successful'.format(file_path=file_path))
                    return package_id, data
                else:
                    message = resp_format.get('message', '')
                    logger.error(u'{file_path}: create the version failed, reason: {message}'.format(
                        file_path=file_path, message=message))
                    sys.exit(1)
            else:
                logger.error(u'{file_path}: create the version failed, status: {status}, message: {message}'.format(
                    file_path=file_path, status=status_code, message=resp.text))
                sys.exit(1)

        except:
            logger.error(u'{file_path}: create the version failed'.format(file_path=file_path))
            logger.error(traceback.format_exc())
            sys.exit(1)

    else:
        logger.error(u'{file_path}: the file is not exist'.format(file_path=file_path))
        sys.exit(1)


def register(file_path, package_id, version_info, version_type, name, remark):
    try:
        headers = HEADER.copy()
        headers['content-type'] = 'application/json'
        headers['Host'] = DEPLOY_HOST
        url = 'http://{DEPLOY_IP}/version/sign'.format(DEPLOY_IP=DEPLOY_IP)

        params_dict = {
            'name': name,
            'memo': remark,
            'sign': version_info['sign'],
            'source': {
                'ip': DEPLOY_REPO_IP,
                'port': 80,
                'host': DEPLOY_REPO_HOST,
                'type': 'http',
                'ensName': DEPLOY_REPO_ENS
            },
            'versionId': version_info['id'],
            'packageId': package_id,
            'env_type': version_type
        }

        resp = requests.post(url=url, json=params_dict, headers=headers, timeout=30)
        if resp.status_code == 200:
            resp_format = resp.json()
            logger.info(u'{file_path}: register successful'.format(file_path=file_path))
        else:
            logger.error(u'{file_path}: register failed, status: {status}, message: {message}'.format(
                file_path=file_path, status=resp.status_code, message=resp.text))
            sys.exit(1)
    except:
        logger.error(u'{file_path}: register failed'.format(file_path=file_path))
        logger.error(traceback.format_exc())
        sys.exit(1)


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='upload the version to the deploy_repository and register to the deploy')
    parser.add_argument('--pkg_name', type=str, help='define the name of the package')
    parser.add_argument('--file', type=str, help='define the path of the upload file')
    parser.add_argument('--remark', type=str, help='the remark of the version')
    parser.add_argument('--config_package', action='store_true', help='set it if it is a config package')
    parser.add_argument('--version', type=str, help='the name of the registered version')
    parser.add_argument('--version_type', type=str, choices=['dev', 'test', 'pre_release', 'prod'], help='the type of the registered version, default test')
    parser.add_argument('--unzip', action='store_true', help='decide the file to be zipped or not, default is unzipped')
    parser.add_argument('--donot_strip_first', action='store_true', help='decide if top directory should be ignored or not, default ignore')
    parser.add_argument('--dockerfile',type=str, default="false",help="upload the dockerfile")
    args = parser.parse_args()
    UNZIP = 'false' if args.unzip else 'true'
    STRIP_FIRST = 'true' if not args.donot_strip_first else 'false'
    if not args.version_type:
        args.version_type = 'test'
    # translate the version type
    if args.version_type == "prod":
        args.version_type = 15
    elif args.version_type == "pre_release":
        args.version_type = 7
    elif args.version_type == "test":
        args.version_type = 3
    else:
        args.version_type = 1
    def usage():
        print (parser.print_help())
        sys.exit(1)

    if args.pkg_name:
        PKG_NAME = args.pkg_name.decode('utf-8')
    else:
        logger.error('the name of the package must be defined')
        usage()

    if args.file:
        FILE = args.file
    else:
        logger.error('the path of the upload file must be defined')
        usage()

    if args.remark:
        REMARK = args.remark.decode('utf-8')
    else:
        logger.error('the remark of the version must be defined')
        usage()

    if args.version:
        VERSION = args.version.decode('utf-8')
    else:
        logger.error('the name of the registered version must be defined')
        usage()
    if args.dockerfile.lower()=="true":
        upload_file=downlod_file(name=PKG_NAME,file=FILE)
        # upload the file to remote deploy_repository
        package_id, version_info = archive(pkg_name=PKG_NAME, file_path=upload_file, remark=REMARK, unzip=UNZIP, strip_first=STRIP_FIRST)

        # register the version
        register(file_path=upload_file, package_id=package_id, version_info=version_info, version_type=args.version_type, name=VERSION, remark=REMARK)
        os.remove(upload_file)
    else:
        package_id, version_info = archive(pkg_name=PKG_NAME, file_path=FILE, remark=REMARK, unzip=UNZIP,
                                           strip_first=STRIP_FIRST)

        # register the version
        register(file_path=FILE, package_id=package_id, version_info=version_info, version_type=args.version_type,
                 name=VERSION, remark=REMARK)