[saltstack]配置Git Fileserver
配置服务器
环境
- salt-master: Salt: 2019.2.0
- Python: 3.6
- 系统: CentOS Linux release 7.6.1810 (Core)
- libgit2: 0.27.3
- pygit2: 0.27.3
安装依赖
# yum install epel-release -y # yum install gcc cmake libffi libffi-devel python36-devel http-parser libssh2 git -y # yum install python-devel
使用yum源内置的libgit2版本有问题，需源码编译安装libgit2
# wget https://github.com/libgit2/libgit2/archive/v0.27.3.zip # unzip v0.27.3.zip # cd libgit2-0.27.3 # mkdir build && cd build # cmake .. # cmake --build . # make install
指定目录安装pygit2
# pip3 install pygit2==0.27.3 -t /usr/lib/python3.6/site-packages
pygit2 0.27.4及更高的版本可能会在被salt调用的时候出现'_pygit2.Reference' object has no attribute 'get_object'错误
查看libgit2是否被识别到
python3
# ldd /usr/lib/python3.6/site-packages/_pygit2.cpython-36m-x86_64-linux-gnu.so linux-vdso.so.1 => (0x00007ffca3f9e000) libgit2.so.27 => not found # 未识别到 libpython3.6m.so.1.0 => /lib64/libpython3.6m.so.1.0 (0x00007f934512e000) libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f9344f12000) libc.so.6 => /lib64/libc.so.6 (0x00007f9344b45000) libdl.so.2 => /lib64/libdl.so.2 (0x00007f9344941000) libutil.so.1 => /lib64/libutil.so.1 (0x00007f934473e000) libm.so.6 => /lib64/libm.so.6 (0x00007f934443c000) /lib64/ld-linux-x86-64.so.2 (0x00007f9345882000)
python2
#ldd /usr/lib/python2.7/site-packages/_pygit2.so linux-vdso.so.1 => (0x00007ffc4d1f2000) libgit2.so.27 => /lib64/libgit2.so.27 (0x00007f14c76fb000) libpython2.7.so.1.0 => /lib64/libpython2.7.so.1.0 (0x00007f14c732e000) libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f14c7112000) libc.so.6 => /lib64/libc.so.6 (0x00007f14c6d4f000) librt.so.1 => /lib64/librt.so.1 (0x00007f14c6b46000) libcurl.so.4 => /lib64/libcurl.so.4 (0x00007f14c68dd000) libssl.so.10 => /lib64/libssl.so.10 (0x00007f14c666b000) libcrypto.so.10 => /lib64/libcrypto.so.10 (0x00007f14c6209000) libz.so.1 => /lib64/libz.so.1 (0x00007f14c5ff3000) libssh2.so.1 => /lib64/libssh2.so.1 (0x00007f14c5dc9000) libdl.so.2 => /lib64/libdl.so.2 (0x00007f14c5bc4000) libutil.so.1 => /lib64/libutil.so.1 (0x00007f14c59c1000) libm.so.6 => /lib64/libm.so.6 (0x00007f14c56bf000) /lib64/ld-linux-x86-64.so.2 (0x0000561bf395f000) libidn.so.11 => /lib64/libidn.so.11 (0x00007f14c548b000) libssl3.so => /lib64/libssl3.so (0x00007f14c5239000) libsmime3.so => /lib64/libsmime3.so (0x00007f14c5012000) libnss3.so => /lib64/libnss3.so (0x00007f14c4ce4000) libnssutil3.so => /lib64/libnssutil3.so (0x00007f14c4ab5000) libplds4.so => /lib64/libplds4.so (0x00007f14c48b1000) libplc4.so => /lib64/libplc4.so (0x00007f14c46ab000) libnspr4.so => /lib64/libnspr4.so (0x00007f14c446d000) libgssapi_krb5.so.2 => /lib64/libgssapi_krb5.so.2 (0x00007f14c4220000) libkrb5.so.3 => /lib64/libkrb5.so.3 (0x00007f14c3f37000) libk5crypto.so.3 => /lib64/libk5crypto.so.3 (0x00007f14c3d04000) libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00007f14c3b00000) liblber-2.4.so.2 => /lib64/liblber-2.4.so.2 (0x00007f14c38f0000) libldap-2.4.so.2 => /lib64/libldap-2.4.so.2 (0x00007f14c369b000) libkrb5support.so.0 => /lib64/libkrb5support.so.0 (0x00007f14c348c000) libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00007f14c3288000) libresolv.so.2 => /lib64/libresolv.so.2 (0x00007f14c306e000) libsasl2.so.3 => /lib64/libsasl2.so.3 (0x00007f14c2e50000) libselinux.so.1 => /lib64/libselinux.so.1 (0x00007f14c2c29000) libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007f14c29f1000) libpcre.so.1 => /lib64/libpcre.so.1 (0x00007f14c278f000) libfreebl3.so => /lib64/libfreebl3.so (0x00007f14c258c000)
如果没有识别到就使用find / -name "libgit2.so.27"查找出路径，再软连接
# ln -s /usr/local/lib/libgit2.so.27 /usr/lib64/libgit2.so.27
# ldconfig -p | grep libgit2 # 成功识别到
libgit2.so.26 (libc6,x86-64) => /lib64/libgit2.so.26
重启salt-master, 可以看到libgit2和pygit2的安装信息
# salt-master --versions-report | grep git
gitdb: Not Installed
gitpython: Not Installed
libgit2: 0.27.3
pygit2: 0.27.3
注意，一定要重启下master，因为虽然不重启-v也能看到有模块了，但是并没有加载进来，使用fileserver命令的时候会不报错，但是一直没有返回数据。
设置FileServer Backend
MasterOfMaster:
/etc/salt/master
default_include: master.d/*.conf order_masters: True # auto_accept: True hash_type: sha256 worker_threads: 4 job_cache: False syndic_wait: 1 #log_level: debug # redis.db: '0' # redis.host: '10.4.231.113' # redis.port: 6379 # master_job_cache: redis
创建配置文件/etc/salt/master.d/fileserver-backend.conf
gitfs_provider: pygit2
fileserver_backend:
- git
gitfs_base: master
gitfs_ref_types:
- branch
gitfs_remotes: # 挂载gitfs目录路径
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: salt
- root: salt
- user: saltstack
- password: xxxxx
- insecure_auth: True
- all_saltenvs: master
- update_interval: 120 # 我们测试发现salt目录不能存放过多的安装文件，会变的非常慢
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: scripts
- root: scripts
- user: saltstack
- password: xxxxx
- insecure_auth: True
- all_saltenvs: master
- mountpoint: salt://scripts
- update_interval: 120
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: reactor
- root: reactor
- user: saltstack
- password: xxxx
- insecure_auth: True
- all_saltenvs: master
- mountpoint: salt://reactor
- update_interval: 600
gitfs_update_interval: 60
git_pillar_provider: pygit2
git_pillar_env: local # 指定pillar/env的环境路径，这边可以用于指定不同环境的名字
ext_pillar:
- git:
- master http://gitlab.ops.net/mlyw/saltstack-files.git:
- root: pillar
- user: saltstack
- password: xxx
- insecure_auth: True
reactor: # 配置自动更新reactor
- salt/fileserver/gitfs/update:
- salt://reactor/update_fileserver.sls
- minion_start:
- salt://reactor/minion_start.sls
syndic
创建配置文件/etc/salt/master.d/fileserver-backend.conf
gitfs_provider: pygit2
fileserver_backend:
- git
- roots
gitfs_base: master
gitfs_ref_types:
- branch
gitfs_remotes:
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: salt
- root: salt
- user: saltstack
- password: xxxxxxxxxxx
- insecure_auth: True
- all_saltenvs: master
- update_interval: 120
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: scripts
- root: scripts
- user: saltstack
- password: xxxxxxxxx
- insecure_auth: True
- all_saltenvs: master
- mountpoint: salt://scripts
- update_interval: 120
- http://gitlab.ops.net/mlyw/saltstack-files.git:
- name: reactor
- root: reactor
- user: saltstack
- password: xxxxxxx
- insecure_auth: True
- all_saltenvs: master
- mountpoint: salt://reactor
- update_interval: 600
git_pillar_env: abc # 每个环境的名称，如jqx/xjq, 对应git仓库中pillar下的目录
git_pillar_provider: pygit2
ext_pillar:
- git:
- master http://gitlab.ops.net/mlyw/saltstack-files.git:
- root: pillar
- user: saltstack
- password: xxxxxx
- insecure_auth: True
reactor:
- minion_start:
- salt://reactor/minion_start.sls
GitLab:
需要将GitLab主机配置为minion， 以及git hook，用来触发master拉取最新文件。当触发post-receive时自动执行脚本(git push)
在git仓库的目录下创建custom_hooks/post-receive脚本，目录一般是/var/opt/gitlab/git-data/repositories/<group>/<project>.git, 如/var/opt/gitlab/git-data/repositories/mlyw/saltstack-files.git/
#!/usr/bin/env sh sudo -u root salt-call event.fire_master update salt/fileserver/gitfs/update
chmod a+x custom_hooks/post-receive
sudo配置:
Cmnd_Alias SALT_GIT_HOOK = /bin/salt-call event.fire_master update salt/fileserver/gitfs/update Defaults!SALT_GIT_HOOK !requiretty ALL ALL=(root) NOPASSWD: SALT_GIT_HOOK
.
├── pillar
│ ├── prodA
│ │ ├── prod_env.sls
│ │ └── top.sls
│ ├── base
│ ├── base_set.sls
│ ├── redis.sls
│ ├── top.sls
├── reactor
│ ├── minion_start.sls
│ ├── update_fileserver.sls
│ └── update_minion.sls
├── salt
│ ├── _returners
│ │ └── redis_pub.py
│ └── _runner
└── scripts
├── app-daemon.sh
reactor/update_fileserver.sls
update_fileserver:
local.cmd.run:
- tgt: 'prod_env:local'
- tgt_type: pillar
- args:
- cmd: salt-run fileserver.update backend=gitfs
sync_all:
local.saltutil.sync_all:
- tgt: '*'
- require:
- salt: update_fileserver
这样配置会造成minion可能刷新不到gitfs的配置：
1 update_fileserver是执行成功的，每个syndic上都刷新到了最新的gitfs
2 minion大部分无法正常获取新值，判断应该是这个两个命令的先后执行顺序问题
3 后面改成如下配置
update_fileserver:
local.cmd.run:
- tgt: 'prod_env:local'
- tgt_type: pillar
- args:
- cmd: salt-run fileserver.update backend=gitfs && salt \* saltutil.sync_all
reactor/minion_start.sls
minion_start:
local.saltutil.sync_all:
- tgt: {{ data['id'] }}
关于event和reactor
salt reactor 使你的基础设施具有反应性和自愈性
1 salt event system
event.send模块： salt ‘*’ sys.doc event.send CLI； salt-call event.send ‘salt/mytag’
2. reactor
对于自定义事件标签的响应，salt-master 捕获信号来操作和执行。
在master配置文件下创建reactor配置文件：
reactor: # Master config section "reactor"
- 'salt/minion/*/start': # Match tag "salt/minion/*/start"
- /srv/reactor/start.sls # Things to do when a minion starts
- /srv/reactor/monitor.sls # Other things to do
- 'salt/cloud/*/destroyed': # Globs can be used to match tags
- /srv/reactor/destroy/*.sls # Globs can be used to match file names
- 'myco/custom/event/tag': # React to custom event tags
- salt://reactor/mycustom.sls # Reactor files can come from the salt fileserver
三种REACTION类型：
- Remote execution，使用salt命令在目标minion上执行一些操作，包括应用state或highstate状态）
- Salt Runners，使用salt-run命令执行的一些任务，如利用http runner去调用一个webhook。
- Wheel，用于管理Salt环境变量、密钥管理或更新配置，如使用salt-key管理认证。
salt事件中event.fire_master 和 event.send的区别
event.send 经过参数判断后还是发送个fire_master
def send(tag,
data=None,
preload=None,
with_env=False,
with_grains=False,
with_pillar=False,
with_env_opts=False,
**kwargs):
'''
Send an event to the Salt Master
.. versionadded:: 2014.7.0
:param tag: A tag to give the event.
......................
if __opts__.get('local') or __opts__.get('file_client') == 'local' or __opts__.get('master_type') == 'disable':
return fire(data_dict, tag)
else:
return fire_master(data_dict, tag, preload=preload)
具体reactor的用法请参照： https://docs.saltstack.com/en/latest/topics/reactor/
常见操作：
- 触发reactor： salt-call event.send salt/fileserver/gitfs/update
- 触发reactor: salt-call event.fire_master update salt/fileserver/gitfs/update
- 手动更新文件： salt-run fileserver.update backend=gitfs
- 从master更新到minion端真正生效会有数秒到一两分钟的延迟，视规模而定
- 查看目录列表： salt-run fileserver.dir_list backend=gitfs
- 查看文件列表： salt-run fileserver.file_list backend=gitfs
- 查看pillar： salt minion_id pillar.item
- 执行state： salt minion_id state.sls {env}.{filename}
- 清空minion的缓存 salt '*' saltutil.clear_cache
- 同步自身缓存： salt-call saltutil.sync_all
- 同步管理的minion缓存: salt '*' saltutil.sync_all
[root@salt-syndic pillar]# salt zijie-test-2 state.sls prod.test
zijie-test-2:
----------
ID: test
Function: cmd.run
Name: echo "I am from prod."
Result: True
Comment: Command "echo "I am from prod."" run
Started: 17:09:43.051629
Duration: 6.472 ms
Changes:
----------
pid:
27107
retcode:
0
stderr:
stdout:
I am from prod. # prod环境
Summary for zijie-test-2
------------
Succeeded: 1 (changed=1)
Failed: 0
------------
Total states run: 1
Total run time: 6.472 ms
[root@salt-syndic pillar]# salt zijie-test-2 state.sls base.test
zijie-test-2:
----------
ID: test
Function: cmd.run
Name: echo "I am from base."
Result: True
Comment: Command "echo "I am from base."" run
Started: 17:10:12.906789
Duration: 5.676 ms
Changes:
----------
pid:
27176
retcode:
0
stderr:
stdout:
I am from base. # base环境
Summary for zijie-test-2
------------
Succeeded: 1 (changed=1)
Failed: 0
------------
Total states run: 1
Total run time: 5.676 ms
- 执行脚本:
salt minion_id cmd.script salt://scripts/{script}， 如：
[root@salt-syndic pillar]# salt 'zijie-test-2' cmd.script salt://scripts/hello.sh
zijie-test-2:
----------
pid:
27341
retcode:
0
stderr:
stdout:
world123!
