harbor高可用集群搭建

高可用harbor集群搭建

一、安装部署

1.节点角色

角色	数量	名称	备注
harbor主节点	2	harbor-1 harbor-2	双主模式
haproxy	2	HA-1 HA-2	需要通过keepalived绑定vip
Redis	高可用集群	redis	harbor组件，本地启动或远程配置
postgresql	高可用集群	postgres	harbor组件，本地启动或远程配置
ceph	高可用集群	ceph	harbor组件

2.离线安装

1、下载离线源码包解压至harbor-1 harbor-2 节点 /data/harbor目录下，修改启动脚本harbor.cfg各项配置参数如下

## Configuration file of Harbor

#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version = 1.7.0
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
hostname = harbor.xxx.xxx.com //配置域名信息

#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.
ui_url_protocol = https //指定为https协议

#Maximum number of job workers in job service 
max_job_workers = 10

#Determine whether or not to generate certificate for the registry's token.
#If the value is on, the prepare script creates new root cert and private key 
#for generating token to access the registry. If the value is off the default key/cert will be used.
#This flag also controls the creation of the notary signer's cert.
customize_crt = on

#The path of cert and key files for nginx, they are applied only the protocol is set to https
ssl_cert = /data/harbor/cert/uat.cmft.com.pem
ssl_cert_key = /data/harbor/cert/uat.cmft.com.pem

#The path of secretkey storage
secretkey_path = /data

#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
admiral_url = NA

#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
log_rotate_count = 50
#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
#are all valid.
log_rotate_size = 200M

#Config http proxy for Clair, e.g. http://my.proxy.com:3128
#Clair doesn't need to connect to harbor internal components via http proxy.
http_proxy =
https_proxy =
no_proxy = 127.0.0.1,localhost,core,registry

#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
#only take effect in the first boot, the subsequent changes of these properties 
#should be performed on web ui

#************************BEGIN INITIAL PROPERTIES************************

#Email account settings for sending out password resetting emails.

#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.
#Identity left blank to act as username.
email_identity =

email_server = smtp.mydomain.com
email_server_port = 25
email_username = sample_admin@mydomain.com
email_password = abc
email_from = admin <sample_admin@mydomain.com>
email_ssl = false
email_insecure = false

##The initial password of Harbor admin, only works for the first time when Harbor starts. 
#It has no effect after the first launch of Harbor.
#Change the admin password from UI after launching Harbor.
harbor_admin_password = yourpassword //配置harboradmin账户密码，建议修改

##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
auth_mode = db_auth

#The url for an ldap endpoint.
ldap_url = ldaps://ldap.mydomain.com

#A user's DN who has the permission to search the LDAP/AD server. 
#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.
#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com

#the password of the ldap_searchdn
#ldap_search_pwd = password

#The base DN from which to look up a user in LDAP/AD
ldap_basedn = ou=people,dc=mydomain,dc=com

#Search filter for LDAP/AD, make sure the syntax of the filter is correct.
#ldap_filter = (objectClass=person)

# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD 
ldap_uid = uid

#the scope to search for users, 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE
ldap_scope = 2

#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
ldap_timeout = 5

#Verify certificate from LDAP server
ldap_verify_cert = true

#The base dn from which to lookup a group in LDAP/AD
ldap_group_basedn = ou=group,dc=mydomain,dc=com

#filter to search LDAP/AD group
ldap_group_filter = objectclass=group

#The attribute used to name a LDAP/AD group, it could be cn, name
ldap_group_gid = cn

#The scope to search for ldap groups. 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE
ldap_group_scope = 2

#Turn on or off the self-registration feature
self_registration = on

#The expiration time (in minute) of token created by token service, default is 30 minutes
token_expiration = 30

#The flag to control what users have permission to create projects
#The default value "everyone" allows everyone to creates a project. 
#Set to "adminonly" so that only admin user can create project.
project_creation_restriction = everyone
#************************END INITIAL PROPERTIES************************

#######Harbor DB configuration section#######

#The address of the Harbor database. Only need to change when using external db.
db_host = 100.12.13.10

#The password for the root user of Harbor DB. Change this before any production use.
db_password = yourdbpassword

#The port of Harbor database host
db_port = 7810

#The user name of Harbor database
db_user = harbordata

#若采用本地配置，则采用如下配置形式，指定db为本地启动。
#db_host = postgresql
#db_password = yourpassword
#db_port = 5432
#db_user = postgres

##### End of Harbor DB configuration#######

##########Redis server configuration.############

#Redis connection address
redis_host = 100.101.21.3

#Redis connection port
redis_port = 4410

#Redis connection password
redis_password = 123456

#Redis connection db index
#db_index 1,2,3 is for registry, jobservice and chartmuseum. 
#db_index 0 is for UI, it's unchangeable
redis_db_index = 1,2,3

########## End of Redis server configuration ############

##########Clair DB configuration############

#Clair DB host address. Only change it when using an exteral DB.
clair_db_host = 100.101.12.3 //clair 使用与db相同的数据库，注意使用新的database，防止数据冲突。
#The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
#Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
clair_db_password = 123456
#Clair DB connect port
clair_db_port = 7810
#Clair DB username
clair_db_username = harbordata
#Clair default database
clair_db = template1 //由于与主数据库公用 不可与db指定相同的database

#clair_db_host = postgresql
#clair_db_password = root123
#clair_db_port = 5432
#clair_db_username = postgres
#clair_db = postgres

#The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
clair_updaters_interval = 12

##########End of Clair DB configuration############

#The following attributes only need to be set when auth mode is uaa_auth
uaa_endpoint = uaa.mydomain.org
uaa_clientid = id
uaa_clientsecret = secret
uaa_verify_cert = true
uaa_ca_cert = /path/to/ca.pem


### Harbor Storage settings ###

#若采用本地ceph 指定 registry_storage_provider_name = filesystem 且忽略registry_storage_provider_config

#默认申请ceph具有如下形式
#bucket region accesskey secretkey regionendpoint参数

#Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.
#registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
registry_storage_provider_name = s3//使用s3协议连接ceph
#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
#To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.
#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
registry_storage_provider_config = bucket: registry, region: default, accesskey: 123456, secretkey: 123456, regionendpoint: http://100.xx.xxx.xxx:7480, rootdirectory: /harbor-registry/st
#ceph建议使用域名配置 若使用ip有可能出现 401 signatureMisMatch的错误。
#registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
#of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signeed certificate.
registry_custom_ca_bundle = 
#If reload_config=true, all settings which present in harbor.cfg take effect after prepare and restart harbor, it overwrites exsiting settings.
#reload_config=false
#Regular expression to match skipped environment variables
#skip_reload_env_pattern=(^EMAIL.*)|(^LDAP.*)

3.初始化与证书生成

1.harbor-1节点/data/harbor目录下执行命令: ./install.sh --with-notary --with-clair --with-chartmuseum #初始化运行harbor watch 'docker ps -a' #监测各组件运行情况 2.拷贝证书 ： /data/harbor/common/config/core/private_key.pem /data/harbor/common/config/registry/root.crt 到/tmp/certfile目录下 同时拷贝到harbor-2节点的相同目录下

4、指定生成证书

#关闭harbor-1节点与相关组件：
docker-compose down -v --remove-orphans 

1.修改启动脚本install.sh 

 if [ -f harbor*.tar.gz ]
 then
 h2 "[Step $item]: loading Harbor images ..."; let item+=1
 #docker load -i ./harbor*.tar.gz
 fi
 echo "" #关闭重复加载image

2.（下列修改操作同时针对 harbor-1 与 harbor-2）

if customize_crt == 'on' and openssl_installed():
print("creating cert-----------------------------")
shell_stat = subprocess.check_call(["which", "openssl"], stdout=FNULL, stderr=subprocess.STDOUT)
empty_subj = "/"
private_key_pem = os.path.join(config_dir, "core", "private_key.pem")
root_crt = os.path.join(config_dir, "registry", "root.crt")
create_root_cert(empty_subj, key_path=private_key_pem, cert_path=root_crt)
mark_file(private_key_pem)
mark_file(root_crt)
else:
print("Copied configuration file: %s" % os.path.join("/tmp", "certfile", "private_key.pem"))
shutil.copyfile(os.path.join("/tmp", "certfile", "private_key.pem"), os.path.join(core_config_dir, "private_key.pem"))
print("Copied configuration file: %s" % os.path.join("/tmp", "certfile", "root.crt"))
shutil.copyfile(os.path.join("/tmp", "certfile", "root.crt"), os.path.join(registry_config_dir, "root.crt"))
#此处将harbor自动生成证书关闭，将证书拷贝到外部路径并直接指定，否则将导致出现两个节点鉴权失败的问题。
修改install.sh 添加参数 customize_crt == off  #关闭自生成证书

5、启动harbor

分别进入harbor-1 harbor-2 执行启动命令

 ./install.sh --with-notary --with-clair --with-chartmuseum 
 watch 'docker ps -a' #监测各组件运行情况

6、配置haproxy/keep-alived

在HA-1 HA-2节点 安装ha/keepalived 修改配置文件如下
harproxy(haproxy.cfg)：

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 20000
user haproxy
group haproxy
daemon
tune.bufsize 131072
tune.ssl.default-dh-param 2048
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlog-normal
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 30m
timeout http-keep-alive 10s
timeout check 10s
maxconn 10000
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
frontend app
bind *:443 
balance roundrobin
mode tcp
server s1 100.65.22.2:443 weight 1  #绑定harbor-1 harbor-2 的443端口
#server s2 100.65.22.3:443 weight 1
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
frontend ceph
bind *:80
balance roundrobin
server app1 100.xx.xx.xx:7480 check  #ceph 可选配置

keepalived (keepalived.conf):

vrrp_script chk_http_haproxy {
script "/app/haproxy/bin/check_haproxy.sh"
interval 10
weight 2
}
global_defs {
notification_email {
admin@admin.com
}
notification_email_from admin@admin.com
smtp_server mail.admin.com
smtp_connect_timeout 30
router_id CMRH_WEB
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 6
priority 101
advert_int 1
track_script {
#run the script 
chk_http_haproxy
}
virtual_ipaddress {
100.70.42.6
}
}


    /app/haproxy/bin/check_haproxy.sh


#! /bin/bash
num=`ps -C haproxy --no-header |wc -l`
if [ $num -eq 0 ]; then
/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg
sleep 3
if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]; then
killall keepalived
fi
fi

#! /bin/bash
num=`ps -C haproxy --no-header |wc -l`
if [ $num -eq 0 ]; then
/app/haproxy/sbin/haproxy -f /app/haproxy/conf/haproxy.cfg
sleep 3
if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]; then
killall keepalived
fi
fi

配置完成后启动在HA-1 HA-2 上启动haproxy

systemcl start keepalived
systemctl start haproxy

7、访问测试

1、访问ha-vip页面 443端口 测试前端访问
2、pull/push测试
3、关闭harbor-1 harbor-2节点：高可用测试

8、常见问题：

1、页面可正常访问，但经常性显示错误，账号登出
haproxy/keepalive配置错误，确认配置
2、harbor启动日志查看/问题排查：/var/log/harbor/下存在的log文件
3、401 SignatureMisMatch ：
ceph地址配置错误，建议使用域名进行配置
4、harbor仓库内部数据访问异常：
此处可能是harbor迁移时，使用的新数据库内部有脏数据导致，可清除template1数据库内所有表数据重置。
5、notary数据库为本地：待解决
6、日志刷新时有概率出现无数据问题：交替访问harbor节点导致，待解决
7、unkown blob问题

When setting up Harbor behind an nginx proxy or elastic load balancing, look for the line below, in common/templates/nginx/nginx.http.conf and remove it from the sections if the proxy already has similar settings: location /, location /v2/ and location /service/.

proxy_set_header X-Forwarded-Proto $scheme;