Harbor双主复制高可用部署
环境信息：
| 主机名称 | IP | 备注 |
|---|---|---|
| harbor01 | 192.168.61.56 | harbor1服务器 |
| harbor02 | 192.168.61.57 | harbor2服务器 |
| 192.168.61.59 | Nginx代理 |
192.168.61.56/57 两个节点分别部署docker-ce ,docker-compose,harbor-offline-installer-v2.9.1.tgz
- 1）部署docker-ce docker-compose
wget -P /etc/yum.repos.d/ https://download.docker.com/linux/centos/docker-ce.repo
yum clean all && yum makecache fast
yum -y install docker-ce docker-compose
systemctl enable docker && systemctl start docker
- 2）HTTPS证书自签
生成CA证书私钥
[root@k8s-harbor01 ~]# openssl genrsa -out ca.key 4096
生成CA证书
[root@k8s-harbor01 ~]# openssl req -x509 -new -nodes -sha512 -days 3650 \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.61.56" \
-key ca.key \
-out ca.crt
生成服务器证书
1）生成私钥
[root@k8s-harbor01 ~]# openssl genrsa -out 192.168.61.56.key 4096
2）生成证书签名请求（CSR）
[root@k8s-harbor01 ~]# openssl req -sha512 -new \
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.61.56" \
-key 192.168.61.56.key \
-out 192.168.61.56.csr
3）生成一个x509 v3扩展文件（两种方式根据情况二选一）
####################################################################################
第一种方式：域名
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1=192.168.61.56
DNS.2=yourdomain
DNS.3=hostname
EOF
####################################################################################
第二种方式：IP
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = IP:192.168.61.56
EOF
####################################################################################
这里选择第二种的IP方式
[root@k8s-harbor01 ~]# cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = IP:192.168.61.56
EOF
4）使用该v3.ext文件为您的Harbor主机生成证书
[root@k8s-harbor01 ~]# openssl x509 -req -sha512 -days 3650 \
-extfile v3.ext \
-CA ca.crt -CAkey ca.key -CAcreateserial \
-in 192.168.61.56.csr \
-out 192.168.61.56.crt
- 3）提供证书给Harbor和Docker
1）将服务器证书和密钥复制到Harbor主机上的certficates文件夹中
根据自己实际环境需求创建Harbor的certficates文件夹
[root@k8s-harbor01 ~]# mkdir -p /data/cert/
[root@k8s-harbor01 ~]# cp 192.168.61.56.crt /data/cert/
[root@k8s-harbor01 ~]# cp 192.168.61.56.key /data/cert/
2）转换 192.168.61.56.crt 为192.168.61.56.cert，供Docker使用。
Docker守护程序将.crt文件解释为CA证书，并将.cert文件解释为客户端证书。
[root@k8s-harbor01 ~]# openssl x509 -inform PEM -in 192.168.61.56.crt -out 192.168.61.56.cert
3）将服务器证书，密钥和CA文件复制到Harbor主机上的Docker certificate文件夹中。
记住必须首先创建适当的文件夹
[root@k8s-harbor01 ~]# mkdir -p /etc/docker/certs.d/192.168.61.56/
[root@k8s-harbor01 ~]# cp 192.168.61.56.cert /etc/docker/certs.d/192.168.61.56/
[root@k8s-harbor01 ~]# cp 192.168.61.56.key /etc/docker/certs.d/192.168.61.56/
[root@k8s-harbor01 ~]# cp ca.crt /etc/docker/certs.d/192.168.61.56/
4）重新启动Docker
[root@k8s-harbor01 ~]# systemctl restart docker
[root@k8s-harbor01 ~]# systemctl status docker
5）将名为"ca.crt"的CA证书下载到本地电脑，然后安装证书。
这样就可以在本地电脑的浏览器里正常访问https地址的Harbor了（证书可被信任）
- 4）离线部署 harbor
# 下载离线部署包
wget https://github.com/goharbor/harbor/releases/download/v2.9.1/harbor-offline-installer-v2.9.1.tgz
tar -zxvf harbor-offline-installer-v2.9.1.tgz
cd harbor
# 修改配置
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 192.168.61.56 # 修改为当前主机IP地址
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /opt/key/server.crt #配置https 证书，证书需要生成
private_key: /opt/key/server.key
# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
# # set enabled to true means internal tls is enabled
# enabled: true
# # put your cert and key files on dir
# dir: /etc/harbor/tls/internal
# # enable strong ssl ciphers (default: false)
# strong_ssl_ciphers: false
# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
external_url: https://192.168.61.56 # 配置访问地址
# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: Harbor12345 # 修改默认账户admin 的登录密码
# 其他配置可按需修改，完成以上配置后开始部署harbor
[root@harbor01 harbor]# ./install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 24.0.7
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.21.0
[Step 2]: loading Harbor images ...
4349dad1c75c: Loading layer [==================================================>] 10.89MB/10.89MB
b3619b11127a: Loading layer [==================================================>] 3.584kB/3.584kB
7b38a647487c: Loading layer [==================================================>] 2.56kB/2.56kB
af63cee918f4: Loading layer [==================================================>] 44.69MB/44.69MB
087e0e1d01ed: Loading layer [==================================================>] 45.48MB/45.48MB
Loaded image: goharbor/harbor-jobservice:v2.9.1
7a18cae000bb: Loading layer [==================================================>] 7.873MB/7.873MB
6e8443cb30ea: Loading layer [==================================================>] 4.096kB/4.096kB
7e6f1921b03b: Loading layer [==================================================>] 17.4MB/17.4MB
7ab103a3c9e9: Loading layer [==================================================>] 3.072kB/3.072kB
afdeb50007ba: Loading layer [==================================================>] 32.78MB/32.78MB
0838cfee6fc3: Loading layer [==================================================>] 50.97MB/50.97MB
Loaded image: goharbor/harbor-registryctl:v2.9.1
06a21a75ac76: Loading layer [==================================================>] 10.89MB/10.89MB
7ef38c74aa21: Loading layer [==================================================>] 3.584kB/3.584kB
78e12b1b294b: Loading layer [==================================================>] 2.56kB/2.56kB
7415bb76ee07: Loading layer [==================================================>] 58.23MB/58.23MB
c25cd46ee82a: Loading layer [==================================================>] 5.632kB/5.632kB
1a3de5a9a094: Loading layer [==================================================>] 122.4kB/122.4kB
2d2d193fbdcd: Loading layer [==================================================>] 80.38kB/80.38kB
ee47e7543fea: Loading layer [==================================================>] 59.23MB/59.23MB
6ebe4d7b431b: Loading layer [==================================================>] 2.56kB/2.56kB
Loaded image: goharbor/harbor-core:v2.9.1
7d9f76d29c1c: Loading layer [==================================================>] 124.4MB/124.4MB
50cb5ae20a44: Loading layer [==================================================>] 3.584kB/3.584kB
59a78c21122c: Loading layer [==================================================>] 3.072kB/3.072kB
6beb01cc5baa: Loading layer [==================================================>] 2.56kB/2.56kB
b09018e5a73f: Loading layer [==================================================>] 3.072kB/3.072kB
c1078fb9f5c7: Loading layer [==================================================>] 3.584kB/3.584kB
59dadef71b1b: Loading layer [==================================================>] 20.48kB/20.48kB
Loaded image: goharbor/harbor-log:v2.9.1
d1de629330a4: Loading layer [==================================================>] 60.48MB/60.48MB
3cb3537a6da7: Loading layer [==================================================>] 173.9MB/173.9MB
a327f18369ed: Loading layer [==================================================>] 25.46MB/25.46MB
3f5c8182a7a2: Loading layer [==================================================>] 63.37MB/63.37MB
a972da56e974: Loading layer [==================================================>] 5.12kB/5.12kB
2fa2fe9c942b: Loading layer [==================================================>] 6.144kB/6.144kB
53214b04b836: Loading layer [==================================================>] 3.072kB/3.072kB
7fde99a5b238: Loading layer [==================================================>] 2.048kB/2.048kB
b985c63bb4f0: Loading layer [==================================================>] 2.56kB/2.56kB
58e8be9a88e4: Loading layer [==================================================>] 7.68kB/7.68kB
Loaded image: goharbor/harbor-db:v2.9.1
64966afbcdad: Loading layer [==================================================>] 10.89MB/10.89MB
7ffad26c4cb7: Loading layer [==================================================>] 27.62MB/27.62MB
ce5f177604e5: Loading layer [==================================================>] 4.608kB/4.608kB
4c46c82379dc: Loading layer [==================================================>] 28.41MB/28.41MB
Loaded image: goharbor/harbor-exporter:v2.9.1
f2d51adf2664: Loading layer [==================================================>] 60.48MB/60.48MB
962114f3c6f4: Loading layer [==================================================>] 110.8MB/110.8MB
fa9bce70fee8: Loading layer [==================================================>] 3.072kB/3.072kB
034a19d4e2af: Loading layer [==================================================>] 59.9kB/59.9kB
ea8a227a5ce4: Loading layer [==================================================>] 61.95kB/61.95kB
Loaded image: goharbor/redis-photon:v2.9.1
b99018c986e9: Loading layer [==================================================>] 115.1MB/115.1MB
Loaded image: goharbor/nginx-photon:v2.9.1
7e85bea4f9eb: Loading layer [==================================================>] 7.873MB/7.873MB
2d7031c02133: Loading layer [==================================================>] 4.096kB/4.096kB
f2f29a057a0e: Loading layer [==================================================>] 3.072kB/3.072kB
c465d9ffbcae: Loading layer [==================================================>] 17.4MB/17.4MB
e8392e2d1c5c: Loading layer [==================================================>] 18.19MB/18.19MB
Loaded image: goharbor/registry-photon:v2.9.1
0ee5099becd4: Loading layer [==================================================>] 8.424MB/8.424MB
709c954772b5: Loading layer [==================================================>] 4.096kB/4.096kB
4b4592e67634: Loading layer [==================================================>] 3.072kB/3.072kB
db64864bc2d4: Loading layer [==================================================>] 196.4MB/196.4MB
b2e16fced657: Loading layer [==================================================>] 14.21MB/14.21MB
261777ce207b: Loading layer [==================================================>] 211.4MB/211.4MB
Loaded image: goharbor/trivy-adapter-photon:v2.9.1
Loaded image: goharbor/prepare:v2.9.1
d0dcb5740755: Loading layer [==================================================>] 115.1MB/115.1MB
a68394b34761: Loading layer [==================================================>] 6.46MB/6.46MB
e47863752870: Loading layer [==================================================>] 245.8kB/245.8kB
eb0d64571e29: Loading layer [==================================================>] 1.233MB/1.233MB
Loaded image: goharbor/harbor-portal:v2.9.1
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /root/harbor
Error happened in config validation...
ERROR:root:Please specify hostname
[root@harbor01 harbor]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
[root@harbor01 harbor]# vi harbor.yml
[root@harbor01 harbor]# vi harbor.yml
[root@harbor01 harbor]# sh install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 24.0.7
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.21.0
[Step 2]: loading Harbor images ...
Loaded image: goharbor/harbor-jobservice:v2.9.1
Loaded image: goharbor/harbor-registryctl:v2.9.1
Loaded image: goharbor/harbor-core:v2.9.1
Loaded image: goharbor/harbor-log:v2.9.1
Loaded image: goharbor/harbor-db:v2.9.1
Loaded image: goharbor/harbor-exporter:v2.9.1
Loaded image: goharbor/redis-photon:v2.9.1
Loaded image: goharbor/nginx-photon:v2.9.1
Loaded image: goharbor/registry-photon:v2.9.1
Loaded image: goharbor/trivy-adapter-photon:v2.9.1
Loaded image: goharbor/prepare:v2.9.1
Loaded image: goharbor/harbor-portal:v2.9.1
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /root/harbor
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
Note: stopping existing Harbor instance ...
[Step 5]: starting Harbor ...
[+] Running 10/10
✔ Network harbor_harbor Created 0.3s
✔ Container harbor-log Started 0.4s
✔ Container registryctl Started 0.5s
✔ Container harbor-portal Started 0.7s
✔ Container redis Started 0.6s
✔ Container registry Started 0.6s
✔ Container harbor-db Started 0.4s
✔ Container harbor-core Started 0.2s
✔ Container nginx Started 0.5s
✔ Container harbor-jobservice Started 0.3s
✔ ----Harbor has been installed and started successfully.----
配置双主复制
测试推送镜像主机配置/etc/docker/daemon.json
# cat /etc/docker/daemon.json
{
"insecure-registries": ["192.168.61.56","192.168.61.57"]
}
systemctl daemon-reload && systemctl restart docekr
两个节点创建项目 kubeflow项目
-
在192.168.61.57 上配置，系统管理-仓库管理-新建目标
-
在192.168.61.57 上配置，系统管理-复制管理-新建规则，也可以使用push-based 触发模式使用事件触发，这里演示使用pull-based
-
在192.168.61.56 上配置，系统管理-仓库管理-新建目标
-
在192.168.61.56 上配置，系统管理-复制管理-新建规则，也可以使用push-based 触发模式使用事件触发，这里演示使用pull-based
测试推送镜像 两节点同步情况
-
推送镜像至192.168.61.57节点仓库
192.168.61.56 节点查看
-
推送镜像至192.168.61.56节点仓库
192.168.61.57 节点查看
-
客户端登录Harbor
在Habror客户端机器（如k8s的node节点、harbor节点）配置登录：
默认情况下，在客户端登录Habor是会报错的：
[root@k8s-node01 ~]# docker login 192.168.61.56
Authenticating with existing credentials...
Login did not succeed, error: Error response from daemon: Get https://172.16.60.238/v2/: x509: certificate signed by unknown authority
原因： 客户端登录Harbor，https证书不被信任。
解决办法：下面两种方法选其一
1）方法一
将Harbor服务器证书，密钥和CA文件复制到Harbor客户主机上的Docker certificate文件夹中
[root@k8s-node01 ~]# mkdir -p /etc/docker/certs.d/192.168.61.56/
[root@k8s-node01 ~]# cd /etc/docker/certs.d/192.168.61.56/
[root@k8s-node01 192.168.61.56]# rsync -e "ssh -p22" -avpgolr 192.168.61.56:/etc/docker/certs.d/192.168.61.56/* ./
[root@k8s-node01 192.168.61.56]# ll
total 12
-rw-r--r-- 1 root root 2053 Aug 19 14:34 172.16.60.238.cert
-rw-r--r-- 1 root root 3243 Aug 19 14:34 172.16.60.238.key
-rw-r--r-- 1 root root 2033 Aug 19 14:34 ca.crt
重启docker服务
[root@k8s-node01 192.168.61.56]# systemctl restart docker
[root@k8s-node01 192.168.61.56]# systemctl status docker
再次验证登录harbor
[root@k8s-node01 192.168.61.56]# docker login 192.168.61.56
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
2）方法二
配置docker服务的daemon.json文件，添加"insecure-registries"参数，表示忽略ssl证书认证。
[root@k8s-node01 ~]# vim /etc/docker/daemon.json
........
"insecure-registries": ["https://192.168.61.56"],
重启docker服务
[root@k8s-node01 ~]# systemctl restart docker
[root@k8s-node01 ~]# systemctl status docker
再次验证登录harbor
[root@k8s-node01 ~]# docker login 192.168.61.56
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
========================================================================
另外，注意客户端机器登录Harbor时，只要首次登录需要输入用户名和密码。
登录成功后的信息默认保存到/root/.docker/config.json文件里。
下次登录时就不用再输入harbor用户名和密码了，直接读取config.json文件内容
[root@k8s-node01 ~]# cat /root/.docker/config.json
{
"auths": {
"192.168.61.56": {
"auth": "YWRtaW46SGFyYm9yQDEyMzQ1Ng=="
}
},
"HttpHeaders": {
"User-Agent": "Docker-Client/18.09.6 (linux)"
}
[root@k8s-node01 ~]# docker login 192.168.61.56
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
作者：一毛
本博客所有文章仅用于学习、研究和交流目的，欢迎非商业性质转载。
不管遇到了什么烦心事，都不要自己为难自己；无论今天发生多么糟糕的事，都不应该感到悲伤。记住一句话：越努力，越幸运。
浙公网安备 33010602011771号