client配置[服务器资产采集,客户端代码]
client 配置
配置文件整合
一般服务的配置文件,分为用户配置文件和系统默认全局配置文件,程序在运行过程中,会优先读取用户的配置文件,并覆盖系统默认配置文件,来实现,用户自定义配置.
client客户端
我们在conf目录下建立 settings 文件(用户配置文件),在 lib 下的 config 目录下建立 global_settings 文件(系统默认全局配置)
# conf 下的 settings 文件中为正常的参数配置
DEBUG = True
API = 'http://127.0.0.1:8000/api/server.html'
# lib/config/__Init__文件,__Init__文件会在导入父文件夹时被执行
import os
import importlib
# 这里我们先导入当前文件夹下的系统默认配置文件
from . import global_settings
class Settings(object):
# 类被实例化时执行
def __init__(self):
# 循环遍历系统配置文件,获取参数名
for item in dir(global_settings):
# 判断如果参数名为大写,则通过 getattr 获取参数值,使用 setattr 来设置为类的属性
if item.isupper():
k = item
v = getattr(global_settings, item)
setattr(self, k, v)
# 通过 environ 来保存用户配置文件的文件位置
settings_path = os.environ.get('AUTO_CLIENT_SETTINGS')
# 通过 importlib 模块使用文件路径导入
md_settings = importlib.import_module(settings_path)
# 循环用户配置文件同上面循环系统配置文件一致
for item in dir(md_settings):
if item.isupper():
k = item
v = getattr(md_settings, item)
# 在设置重复属性时,会覆盖掉系统配置文件的参数配置,因为我们先循环系统配置文件
setattr(self, k, v)
# 实例化这个类
settings = Settings()
# 客户端启动程序,这里
import requests
import subprocess
import os
import sys
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASEDIR)
# 客户端程序运行时,使用 environ 保存了客户端文件的位置
os.environ['AUTO_CLIENT_SETTINGS'] = 'conf.settings'
if __name__ == '__main__':
# 直接导入 lib.config 中我们实例化的settings
from lib.config import settings
print(settings.DEBUG)
print(settings.API)
# 运行结果,看到用户配置文件中 DEBUG 覆盖了系统配置文件的参数
C:\Users\Gldsly\AppData\Local\Programs\Python\Python36\python.exe E:/autosystem_project/client/bin/run.py
True
http://127.0.0.1:8000/api/server.html
Process finished with exit code 0
服务端资源模块整合
客户端程序的所有的信息采集模块都是独立工作的,我们可以通过
__init__文件来同一管理所有的模块,通过 settings 文件中的 PLUGIN_ITEMS 中设置来调用相应的模块进行信息采集工作.
# run.py 客户端启动程序
import os
import sys
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASEDIR)
os.environ['AUTO_CLIENT_SETTINGS'] = 'conf.settings'
from src import script
if __name__ == '__main__':
# 通过 script.py 来选择客户端的启动模式 AGENT / SSH / SALT
script.start()
# script.py 通过 settings 中的参数配置启动对应的客户端
from lib.config import settings
from .client import AgentClient
from .client import SaltSshClient
def start():
if settings.MODE == 'AGENT':
obj = AgentClient()
elif settings.MODE == 'SSH' or settings.MODE == 'SALT':
obj = SaltSshClient()
else:
raise Exception('模式不存在')
obj.exec()
# client.py 设置各种模式的启动方式, script.py 文件调用这里的类来启动对应模式的客户端
from src.plugins import PluginManager
from lib.config import settings
# 导入了线程池模块
from concurrent.futures import ThreadPoolExecutor
import requests
# 所有模式继承的基础类,共有的功能在这里
class BaseClient(object):
def __init__(self):
self.api = settings.API
# 上传数据给服务端方法,数据保存在 request.body 中
def post_server_info(self,server_info):
response = requests.post(self.api, json=server_info)
# 这里括号内的 json=XX 会序列化字段信息,并加上请求头
print(response)
# Agent模式客户端
class AgentClient(BaseClient):
def exec(self):
obj = PluginManager()
server_info = obj.exec_plugin()
print('采集到的服务器信息:',server_info)
self.post_server_info(server_info)
# Salt 或者 SSH 客户端
class SaltSshClient(BaseClient):
# 实际客户端启动代码被下面的线程调用
def task(self,host):
obj = PluginManager(host)
server_info = obj.exec_plugin()
print('采集到的服务器信息:', server_info)
self.post_server_info(server_info)
def get_host(self):
return ['c1.com','c2.com']
# 这里使用了线程池
def exec(self):
pool = ThreadPoolExecutor(10)
for host in self.get_host():
pool.submit(self.task,host)
# scr.plugins 下的 __init__.py 文件, client.py 中调用这个文件中PluginManager 的来执行对应的模块
from lib.config import settings
import importlib
import traceback
# 管理采集信息模块的类
class PluginManager(object):
def __init__(self,hostname=None):
self.plugin_items = settings.PLUGIN_ITEMS
self.hostname = hostname
self.debug = settings.DEBUG
if settings.MODE == 'SSH':
self.ssh_user = settings.SSH_USER
self.ssh_pwd = settings.SSH_PWD
self.ssh_port = settings.SSH_PORT
# 执行模块的方法
def exec_plugin(self):
server_info = {}
# 获取 settings 文件中配置的需要采集的信息,通过路径导入对应的模块
for k, v in settings.PLUGIN_ITEMS.items():
# 初始化定义一个 info 字典,用于存储模块的采集信息
info = {'status':True,'data': None,'msg':None}
try:
# v 是模块路径和模块文件中的类名,这里做切分
module_path, cls_name = v.rsplit('.', maxsplit=1)
# 导入模块文件
module = importlib.import_module(module_path)
# 获取模块文件中的类
cls = getattr(module, cls_name)
# 在执行模块前,判断是否有 initial 方法,此方法用于在执行模块采集信息之前做一些,自定义操作
if hasattr(cls, 'initial'):
obj = cls.initial()
else:
obj = cls()
# 执行模块中的方法,收集信息,这里传入执行信息采集的命令方法和是否是测试模式
ret = obj.process(self.exec_cmd,self.debug)
# 返回值是放到 info 的 data 中
info['data'] = ret
except Exception as e:
# 发生错误时 info 的 status 设置为 False
info['status'] = False
# 错误信息写入 msg 中
info['msg'] = traceback.format_exc()
# 把模块的名字和信息对应写入 server_info 中
server_info[k] = info
return server_info
# 客户端执行采集命令的处理方法,使用 if 判断三种模式,模式配置在 settings 中
def exec_cmd(self,cmd):
if settings.MODE == 'AGENT':
import subprocess
result = subprocess.getoutput(cmd)
elif settings.MODE == 'SSH':
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=self.hostname, port=self.ssh_port, username=self.ssh_user, password=self.ssh_pwd)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stdout.read()
ssh.close()
elif settings.MODE == 'SALT':
import subprocess
result = subprocess.getoutput('salt "%s" cmd.run %s' % (self.hostname,cmd))
else:
raise Exception('模式错误')
# 最后返回的是命令处理结果
return result
以下为各个模块的具体代码,每个模块都有个 if 判断是否是 DEBUG 模式,else 为正式生产环境中代码,这里代码为 copy 的代码
# 基本信息,操作系统的基本系统
class Basic(object):
@classmethod
def initial(cls):
return cls()
def process(self,cmd_func,debug):
if debug:
output = {
'os_platform': "linux",
'os_version': "CentOS release 6.6 (Final)\nKernel \r on an \m",
'hostname': 'c1.com'
}
else:
output = {
'os_platform': cmd_func("uname").strip(),
'os_version': cmd_func("cat /etc/issue").strip().split('\n')[0],
'hostname': cmd_func("hostname").strip(),
}
return output
# 主板信息,采集主板的相关信息
import os
from lib.config import settings
class Board(object):
def process(self, cmd_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/board.out'), 'r', encoding='utf-8').read()
else:
output = cmd_func("sudo dmidecode -t1")
return self.parse(output)
def parse(self, content):
result = {}
key_map = {
'Manufacturer': 'manufacturer',
'Product Name': 'model',
'Serial Number': 'sn',
}
for item in content.split('\n'):
row_data = item.strip().split(':')
if len(row_data) == 2:
if row_data[0] in key_map:
result[key_map[row_data[0]]] = row_data[1].strip() if row_data[1] else row_data[1]
return result
# 硬盘信息,采集硬盘的信息
import sys
import os
import re
from lib.config import settings
class Disk(object):
def __init__(self):
pass
@classmethod
def settings(cls):
return cls()
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/disk.out'), 'r', encoding='utf-8').read()
else:
output = command_func("sudo MegaCli -PDList -aALL")
return self.parse(output)
def parse(self, content):
response = {}
result = []
for row_line in content.split("\n\n\n\n"):
result.append(row_line)
for item in result:
temp_dict = {}
for row in item.split('\n'):
if not row.strip():
continue
if len(row.split(':')) != 2:
continue
key, value = row.split(':')
name = self.mega_patter_match(key)
if name:
if key == 'Raw Size':
raw_size = re.search('(\d+\.\d+)', value.strip())
if raw_size:
temp_dict[name] = raw_size.group()
else:
raw_size = '0'
else:
temp_dict[name] = value.strip()
if temp_dict:
response[temp_dict['slot']] = temp_dict
return response
@staticmethod
def mega_patter_match(needle):
grep_pattern = {'Slot': 'slot', 'Raw Size': 'capacity', 'Inquiry': 'model', 'PD Type': 'pd_type'}
for key, value in grep_pattern.items():
if needle.startswith(key):
return value
return False
# 内存信息,采集内存相关信息
import os
from lib.config import settings
from lib import convert
class Memory(object):
def process(self, cmd_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/memory.out'), 'r', encoding='utf-8').read()
else:
output = cmd_func("sudo dmidecode -q -t 17 2>/dev/null")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
ram_dict = {}
key_map = {
'Size': 'capacity',
'Locator': 'slot',
'Type': 'model',
'Speed': 'speed',
'Manufacturer': 'manufacturer',
'Serial Number': 'sn',
}
devices = content.split('Memory Device')
for item in devices:
item = item.strip()
if not item:
continue
if item.startswith('#'):
continue
segment = {}
lines = item.split('\n\t')
for line in lines:
if not line.strip():
continue
if len(line.split(':')):
key, value = line.split(':')
else:
key = line.split(':')[0]
value = ""
if key in key_map:
if key == 'Size':
segment[key_map['Size']] = convert.convert_mb_to_gb(value, 0)
else:
segment[key_map[key.strip()]] = value.strip()
ram_dict[segment['slot']] = segment
return ram_dict
# 网卡信息,采集网卡的相关信息
import os
import re
from lib.config import settings
class Network(object):
def process(self, cmd_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/nic.out'), 'r', encoding='utf-8').read()
interfaces_info = self._interfaces_ip(output)
else:
interfaces_info = self.linux_interfaces(cmd_func)
self.standard(interfaces_info)
return interfaces_info
def linux_interfaces(self, command_func):
'''
Obtain interface information for *NIX/BSD variants
'''
ifaces = dict()
ip_path = 'ip'
if ip_path:
cmd1 = command_func('sudo {0} link show'.format(ip_path))
cmd2 = command_func('sudo {0} addr show'.format(ip_path))
ifaces = self._interfaces_ip(cmd1 + '\n' + cmd2)
return ifaces
def which(self, exe):
def _is_executable_file_or_link(exe):
# check for os.X_OK doesn't suffice because directory may executable
return (os.access(exe, os.X_OK) and
(os.path.isfile(exe) or os.path.islink(exe)))
if exe:
if _is_executable_file_or_link(exe):
# executable in cwd or fullpath
return exe
# default path based on busybox's default
default_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
search_path = os.environ.get('PATH', default_path)
path_ext = os.environ.get('PATHEXT', '.EXE')
ext_list = path_ext.split(';')
search_path = search_path.split(os.pathsep)
if True:
# Add any dirs in the default_path which are not in search_path. If
# there was no PATH variable found in os.environ, then this will be
# a no-op. This ensures that all dirs in the default_path are
# searched, which lets salt.utils.which() work well when invoked by
# salt-call running from cron (which, depending on platform, may
# have a severely limited PATH).
search_path.extend(
[
x for x in default_path.split(os.pathsep)
if x not in search_path
]
)
for path in search_path:
full_path = os.path.join(path, exe)
if _is_executable_file_or_link(full_path):
return full_path
return None
def _number_of_set_bits_to_ipv4_netmask(self, set_bits): # pylint: disable=C0103
'''
Returns an IPv4 netmask from the integer representation of that mask.
Ex. 0xffffff00 -> '255.255.255.0'
'''
return self.cidr_to_ipv4_netmask(self._number_of_set_bits(set_bits))
def cidr_to_ipv4_netmask(self, cidr_bits):
'''
Returns an IPv4 netmask
'''
try:
cidr_bits = int(cidr_bits)
if not 1 <= cidr_bits <= 32:
return ''
except ValueError:
return ''
netmask = ''
for idx in range(4):
if idx:
netmask += '.'
if cidr_bits >= 8:
netmask += '255'
cidr_bits -= 8
else:
netmask += '{0:d}'.format(256 - (2 ** (8 - cidr_bits)))
cidr_bits = 0
return netmask
def _number_of_set_bits(self, x):
'''
Returns the number of bits that are set in a 32bit int
'''
# Taken from http://stackoverflow.com/a/4912729. Many thanks!
x -= (x >> 1) & 0x55555555
x = ((x >> 2) & 0x33333333) + (x & 0x33333333)
x = ((x >> 4) + x) & 0x0f0f0f0f
x += x >> 8
x += x >> 16
return x & 0x0000003f
def _interfaces_ip(self, out):
'''
Uses ip to return a dictionary of interfaces with various information about
each (up/down state, ip address, netmask, and hwaddr)
'''
ret = dict()
right_keys = ['name', 'hwaddr', 'up', 'netmask', 'ipaddrs']
def parse_network(value, cols):
'''
Return a tuple of ip, netmask, broadcast
based on the current set of cols
'''
brd = None
if '/' in value: # we have a CIDR in this address
ip, cidr = value.split('/') # pylint: disable=C0103
else:
ip = value # pylint: disable=C0103
cidr = 32
if type_ == 'inet':
mask = self.cidr_to_ipv4_netmask(int(cidr))
if 'brd' in cols:
brd = cols[cols.index('brd') + 1]
return (ip, mask, brd)
groups = re.compile('\r?\n\\d').split(out)
for group in groups:
iface = None
data = dict()
for line in group.splitlines():
if ' ' not in line:
continue
match = re.match(r'^\d*:\s+([\w.\-]+)(?:@)?([\w.\-]+)?:\s+<(.+)>', line)
if match:
iface, parent, attrs = match.groups()
if 'UP' in attrs.split(','):
data['up'] = True
else:
data['up'] = False
if parent and parent in right_keys:
data[parent] = parent
continue
cols = line.split()
if len(cols) >= 2:
type_, value = tuple(cols[0:2])
iflabel = cols[-1:][0]
if type_ in ('inet',):
if 'secondary' not in cols:
ipaddr, netmask, broadcast = parse_network(value, cols)
if type_ == 'inet':
if 'inet' not in data:
data['inet'] = list()
addr_obj = dict()
addr_obj['address'] = ipaddr
addr_obj['netmask'] = netmask
addr_obj['broadcast'] = broadcast
data['inet'].append(addr_obj)
else:
if 'secondary' not in data:
data['secondary'] = list()
ip_, mask, brd = parse_network(value, cols)
data['secondary'].append({
'type': type_,
'address': ip_,
'netmask': mask,
'broadcast': brd,
})
del ip_, mask, brd
elif type_.startswith('link'):
data['hwaddr'] = value
if iface:
if iface.startswith('pan') or iface.startswith('lo') or iface.startswith('v'):
del iface, data
else:
ret[iface] = data
del iface, data
return ret
def standard(self, interfaces_info):
for key, value in interfaces_info.items():
ipaddrs = set()
netmask = set()
if not 'inet' in value:
value['ipaddrs'] = ''
value['netmask'] = ''
else:
for item in value['inet']:
ipaddrs.add(item['address'])
netmask.add(item['netmask'])
value['ipaddrs'] = '/'.join(ipaddrs)
value['netmask'] = '/'.join(netmask)
del value['inet']
下面是 settings 中的参数配置
# settings.py
import os
MODE = 'SSH'
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SSH_USER = 'root'
SSH_PWD = 'password'
SSH_PORT = 22
DEBUG = True
API = 'http://127.0.0.1:8000/api/server.html'
PLUGIN_ITEMS = {
"disk": "src.plugins.disk.Disk",
"basic": "src.plugins.basic.Basic",
"board": "src.plugins.board.Board",
"memory": "src.plugins.mem.Memory",
"network": "src.plugins.network.Network",
}
客户端采集信息的三种方式
1. agent形式
import subprocess
result = subprocess.getoutput("命令")
2. SSH模式
pip3 install paramiko
import paramiko
192.168.16.72
root
redhat
用户名和密码方式
import paramiko
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname='192.168.16.72', port=22, username='root', password='redhat')
stdin, stdout, stderr = ssh.exec_command('ifconfig')
result = stdout.read()
ssh.close()
print(result)
公钥私钥方式
import paramiko
private_key = paramiko.RSAKey.from_private_key_file('/home/auto/.ssh/id_rsa')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname='c1.salt.com', port=22, username='wupeiqi', pkey=private_key)
stdin, stdout, stderr = ssh.exec_command('df')
result = stdout.read()
ssh.close()
3. SaltStack模式
import subprocess
subprocess.getoutput('salt "c1.com" cmd.run "命令"')
浙公网安备 33010602011771号