网络质量IP获取脚本

 

 

安装python3，并安装所需插件

 yum -y install epel-release ; yum install -y python36 python36-devel; curl https://bootstrap.pypa.io/get-pip.py |python3.6

vi ~/.pip/pip.conf
[global] 
index-url = https://pypi.tuna.tsinghua.edu.cn/simple
[install]
trusted-host = https://pypi.tuna.tsinghua.edu.cn

 

[root@localhost]# more requirements.txt
retrying==1.3.3
IPy==1.0
lxml==4.3.4
pycurl==7.43.0.3

ps：安装可能会出现如下错误：

安装：yum install  curl-devel gcc -y

2,无法引入pycurl

 

 

 解决：

 pip3 uninstall pycurl ; export PYCURL_SSL_LIBRARY=nss ; pip3 install --ignore-installed pycurl

备注：如果uninstall无法移除，则手动移除
mv /usr/local/python3/lib/python3.6/site-packages/pycurl* /home

日志插件脚本： common.py

import logging
import logging.handlers


class LogHandler(object):
    FILE_NAME = None
    def __init__(self, name):
        self.handler = logging.handlers.RotatingFileHandler('/tmp/idc_ping_monitor.log' , maxBytes = 1024*1024 , backupCount = 5)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
        self.handler.setFormatter(formatter)
        self.logger = logging.getLogger(name)
        self.logger.addHandler(self.handler)

    def __call__(self , func , *args , **kwargs):
        if func.__name__ == 'info' :
            self.logger.setLevel(logging.INFO)
            log_record = func(*args , **kwargs)
            self.logger.info(log_record)
            self.logger.removeHandler(self.handler)

　　

IP获取脚本：getip.py

#-*- coding:utf-8 -*-
import pycurl
from lxml import etree
from io import BytesIO
import collections
import json
import IPy
import os
import datetime
from retrying import retry
import multiprocessing
from common import LogHandler


def infoInit():
    province_info = collections.defaultdict()

    province_info["beijing"] = {"name" : "北京" , "abbr" : "北京" , "url" : "http://ip.yqie.com/cn/beijing/"}
    province_info["tianjin"] = {"name" : "天津" , "abbr" : "天津" , "url" : "http://ip.yqie.com/cn/tianjin/"}
    province_info["hebei"] = {"name" : "石家庄" , "abbr" : "河北" , "url" : "http://ip.yqie.com/cn/hebei/shijiazhuang/"}
    province_info["neimenggu"] = {"name" : "呼和浩特" , "abbr" : "内蒙" , "url" : "http://ip.yqie.com/cn/neimenggu/huhehaote/"}
    province_info["liaoning"] = {"name" : "沈阳" , "abbr" : "辽宁" , "url" : "http://ip.yqie.com/cn/liaoning/shenyang/"}
    province_info["heilongjiang"] = {"name" : "海尔滨" , "abbr" : "黑龙江" , "url" : "http://ip.yqie.com/cn/heilongjiang/haerbin/"}
    province_info["jilin"] = {"name" : "长春" , "abbr" : "吉林" , "url" : "http://ip.yqie.com/cn/jilin/changchun/"}
    province_info["shandong"] = {"name" : "济南" , "abbr" : "山东" , "url" : "http://ip.yqie.com/cn/shandong/jinan/"}
    province_info["shanxi"] = {"name" : "太原" , "abbr" : "山西" , "url" : "http://ip.yqie.com/cn/sanxi/taiyuan/"}
    province_info["shanghai"] = {"name" : "上海" , "abbr" : "上海" , "url" : "http://ip.yqie.com/cn/shanghai/"}
    province_info["zhejiang"] = {"name" : "杭州" , "abbr" : "浙江" , "url" : "http://ip.yqie.com/cn/zhejiang/hangzhou/"}
    province_info["jiangsu"] = {"name" : "南京" , "abbr" : "江苏" , "url" : "http://ip.yqie.com/cn/jiangsu/nanjing/"}
    province_info["anhui"] = {"name" : "合肥" , "abbr" : "安徽" , "url" : "http://ip.yqie.com/cn/anhui/hefei/"}
    province_info["jiangxi"] = {"name" : "南昌" , "abbr" : "江西" , "url" : "http://ip.yqie.com/cn/jiangxi/nanchang/"}
    province_info["fujian"] = {"name" : "福州" , "abbr" : "福建" , "url" : "http://ip.yqie.com/cn/fujian/fuzhou/"}
    province_info["henan"] = {"name" : "郑州" , "abbr" : "河南" , "url" : "http://ip.yqie.com/cn/henan/zhengzhou/"}
    province_info["hubei"] = {"name" : "武汉" , "abbr" : "湖北" , "url" : "http://ip.yqie.com/cn/hubei/wuhan/"}
    province_info["hunan"] = {"name" : "长沙" , "abbr" : "湖南" , "url" : "http://ip.yqie.com/cn/hunan/changsha/"}
    province_info["guangdong"] = {"name" : "广州" , "abbr" : "广东" , "url" : "http://ip.yqie.com/cn/guangdong/guangzhou/"}
    province_info["guangxi"] = {"name" : "南宁" , "abbr" : "广西" , "url" : "http://ip.yqie.com/cn/guangxi/nanning/"}
    province_info["dongguan"] = {"name" : "东莞" , "abbr" : "广东" , "url" : "http://ip.yqie.com/cn/guangdong/dongguan/"}
    province_info["hainan"] = {"name" : "海口" , "abbr" : "海南" , "url" : "http://ip.yqie.com/cn/hainan/haikou/"}
    province_info["sichuan"] = {"name" : "成都" , "abbr" : "四川" , "url" : "http://ip.yqie.com/cn/sichuan/chengdu/"}
    province_info["chongqing"] = {"name" : "重庆" , "abbr" : "重庆" , "url" : "http://ip.yqie.com/cn/chongqing/"}
    province_info["guizhou"] = {"name" : "贵阳" , "abbr" : "贵州" , "url" : "http://ip.yqie.com/cn/guizhou/guiyang/"}
    province_info["yunnan"] = {"name" : "昆明" , "abbr" : "云南" , "url" : "http://ip.yqie.com/cn/yunnan/kunming/"}
    province_info["xizang"] = {"name" : "拉萨" , "abbr" : "西藏" , "url" : "http://ip.yqie.com/cn/xizang/lasa/"}
    province_info["shaanxi"] = {"name" : "西安" , "abbr" : "陕西" , "url" : "http://ip.yqie.com/cn/shanxi/xian/"}
    province_info["ningxia"] = {"name" : "银川" , "abbr" : "宁夏" , "url" : "http://ip.yqie.com/cn/ningxia/yinchuan/"}
    province_info["gansu"] = {"name" : "兰州" , "abbr" : "甘肃" , "url" : "http://ip.yqie.com/cn/gansu/lanzhou/"}
    province_info["qinghai"] = {"name" : "西宁" , "abbr" : "青海" , "url" : "http://ip.yqie.com/cn/qinghai/xining/"}
    province_info["xinjiang"] = {"name" : "乌鲁木齐" , "abbr" : "新疆" , "url" : "http://ip.yqie.com/cn/xinjiang/wulumuqi/"}

    return province_info



class GetInfo(object):
    
    def __init__(self):
        pass

    @retry(stop_max_attempt_number=1)
    def getHtml(self , province , url , province_info_shared):
        ####
        @LogHandler('getHtml()')
        def info():
            return '[info] get ' + url
        buffer = BytesIO()
        c = pycurl.Curl()
        c.setopt(c.URL, url)
        c.setopt(c.WRITEDATA, buffer)
        try:
            c.perform()
        except Exception as e :
            @LogHandler('getHtml()')
            def error():
                return 'retry ' + url + e
            raise e
        c.close()
        body = buffer.getvalue()
        if province in ('shaanxi' , 'qinghai') :
            body = body[:-2]
            
        html = etree.HTML(body.decode("utf-8","ignore"))
        piece_list = html.xpath('//table[@id="GridViewOrder"]/tr')
        
        results = self.parse(piece_list)
        province_info_temp = province_info_shared[province]
        province_info_temp.update({"cmcc" : results.get('cmcc')[0]})
        province_info_temp.update({"telcom" : results.get('telcom')[0]})
        province_info_temp.update({"unicom" : results.get('unicom')[0]})
        province_info_shared[province] = province_info_temp
        
    def parse(self , piece_list):
        results = collections.defaultdict()
        results['cmcc'] = []
        results['telcom'] = []
        results['unicom'] = []
    
        ip_addr = None
        location = None
    
        for piece in  piece_list:
            ip_start = piece.xpath('./*')[1].text
            ip_end = piece.xpath('./*')[2].text
            location = piece.xpath('./*')[3].text
    
            if results['cmcc'] and results['telcom'] and results['unicom']:
                break
    
            if not results['cmcc'] and '移动' in location :
                (exit_code , ip_addr) = self.pingCheck(ip_start , ip_end)
                if exit_code :
                    results['cmcc'].append(ip_addr)
                    results['cmcc'].append(location)
                    continue
            if not results['telcom'] and '电信' in location :
                (exit_code , ip_addr) = self.pingCheck(ip_start , ip_end)
                if exit_code :
                    results['telcom'].append(ip_addr)
                    results['telcom'].append(location)
                    continue
            if not results['unicom'] and '联通' in location :
                (exit_code , ip_addr) = self.pingCheck(ip_start , ip_end)
                if exit_code :
                    results['unicom'].append(ip_addr)
                    results['unicom'].append(location)
                    continue
            
        return results

    def pingCheck(self , ip_start , ip_end):
        ip_start_int = IPy.IP(ip_start).int()
        ip_end_int = IPy.IP(ip_end).int()
        for_count = 0
        ip_addr = None
        if ip_end_int - ip_start_int > 10 :
            for_count = 10
        else :
            for_count = ip_end_int - ip_start_int + 1
        for count in range(for_count):
            ip_addr = IPy.IP(ip_start_int + count).strNormal(0)
            os_code = not os.system("ping -c 1 -w 1 "+ ip_addr +" > /dev/null") if True else False
            if os_code :
                break
        return (os_code , ip_addr)
        
    
    def getIP(self , province_info):
        pool = multiprocessing.Pool(processes = 4)
        province_info_shared = multiprocessing.Manager().dict()
        province_info_shared.update(province_info)

        for province in province_info_shared:
            pool.apply_async(self.getHtml , (province , province_info_shared.get(province).get("url") , province_info_shared), error_callback=self.throw_error) 
        pool.close()
        pool.join()  

        return province_info_shared

    def throw_error(self , e):
        raise e


def syncToSmokingFile(province_info):
    data = collections.defaultdict()
    data['cmcc'] = "+ CMCC \n" \
                   "menu = 移动 \n" \
                   "title = 移动 \n" \
                   "\n"
    data['unicom'] = "+ UNICOM \n" \
                   "menu = 联通 \n" \
                   "title = 联通 \n" \
                   "\n"
    data['telcom'] = "+ TELCOM \n" \
                   "menu = 电信 \n" \
                   "title = 电信 \n" \
                   "\n"

    for province in province_info:
        if province_info[province].get('cmcc'):
            record = "++ " + province_info[province].get('abbr') + "\n" \
                     "menu = " + province_info[province].get('name') + "\n" \
                     "host = " + province_info[province].get('cmcc') + "\n" \
                     "\n"
            data['cmcc'] += record
        if province_info[province].get('unicom'):
            record = "++ " + province_info[province].get('abbr') + "\n" \
                     "menu = " + province_info[province].get('name') + "\n" \
                     "host = " + province_info[province].get('unicom') + "\n" \
                     "\n"
            data['unicom'] += record
        if province_info[province].get('telcom'):
            record = "++ " + province_info[province].get('abbr') + "\n" \
                     "menu = " + province_info[province].get('name') + "\n" \
                     "host = " + province_info[province].get('telcom') + "\n" \
                     "\n"
            data['telcom'] += record

    for isp in data:
        with open("location/"+isp , "w") as f:
            f.writelines(data[isp])

def getDate():
    return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")


if __name__ == '__main__':
    print(getDate() , 'begin...')
    getinfo = GetInfo()
    province_info = getinfo.getIP(infoInit())
    syncToSmokingFile(province_info)
    print(getDate() , 'end...')

.

拼音问题：

vi collection_to_prometheus.py

#coding:utf-8
import requests
import rrdtool
import os
from common import LogHandler


paras = {
    'province_map' : {
        'anhui'         : '安徽' ,
        'beijing'       : '北京' ,
        'chongqing'     : '重庆' ,
        'fujian'        : '福建' ,
        'gansu'         : '甘肃' ,
        'guangdong'     : '广东' ,
        'guangxi'       : '广西' ,
        'guizhou'       : '贵州' ,
        'hainan'        : '海南' ,
        'hebei'         : '河北' ,
        'heilongjiang'  : '黑龙江' ,
        'henan'         : '河南' ,
        'hubei'         : '湖北' ,
        'hunan'         : '湖南' ,
        'jiangsu'       : '江苏' ,
        'jiangxi'       : '江西' ,
        'jilin'         : '吉林' ,
        'liaoning'      : '辽宁' ,
        'neimenggu'     : '内蒙古' ,
        'ningxia'       : '宁夏' ,
        'qinghai'       : '青海' ,
        'shaanxi'       : '陕西' ,
        'shandong'      : '山东' ,
        'shanghai'      : '上海' ,
        'shanxi'        : '山西' ,
        'shenzhen'      : '深圳' ,
        'sichuan'       : '四川' ,
        'tianjin'       : '天津' ,
        'xinjiang'      : '新疆' ,
        'xizang'        : '西藏' ,
        'yunnan'        : '云南' ,
        'zhejiang'      : '浙江'
    } , 
    'prometheus_gateway' : 'http://192.168.56.101:9091' , 
    'data_dir' : '/usr/local/smokeping/data'
}

def pushMetrics(instance , ISP , key , value):
    headers = {'X-Requested-With': 'Python requests', 'Content-type': 'text/xml'}
    pushgateway = '%s/metrics/job/smokeping-collected-%s/instance/%s' % (paras['prometheus_gateway'] , ISP , instance)
    metrics = 'smokeping_%s{instance=\"%s\" , ISP=\"%s\" , IDC=\"%s\" , alias=\"%s\"} %d' % (key , instance , ISP , 'SH' , paras['province_map'].get(instance) , value)
    request_code = requests.post(pushgateway , data='{0}\n'.format(metrics).encode('utf-8') , headers=headers)
    @LogHandler(pushgateway)
    def info():
        return metrics + ' - ' + str(request_code.status_code)

def getMonitorData(rrd_file): 
    rrd_info = rrdtool.info(rrd_file)
    last_update = rrd_info['last_update'] - 60
    args = '-s ' + str(last_update) 
    results = rrdtool.fetch(rrd_file , 'AVERAGE' , args )
    lost_package_num = int(results[2][0][1])
    average_rrt = 0 if not results[2][0][2] else results[2][0][2] * 1000
    return lost_package_num , round(average_rrt , 4)


if __name__ == '__main__':
    ISP_list = ['TELCOM' , 'CMCC' , 'UNICOM' , 'TENCENT']
    for ISP in ISP_list:
        rrd_data_dir = os.path.join(paras['data_dir'] , ISP)
        for filename in os.listdir(rrd_data_dir):
            (instance , postfix) = os.path.splitext(filename)
            if postfix == '.rrd' :
                (lost_package_num , rrt) = getMonitorData(os.path.join(paras['data_dir'] , ISP , filename))
                pushMetrics(instance , ISP , 'rrt' , rrt)
                pushMetrics(instance , ISP , 'lost_package_num' , lost_package_num)