kubernetes第六次课

kubecolor （命令输出有颜色）

https://github.com/hidetatz/kubecolor

root@k8s-master1:~# wget https://github.com/hidetatz/kubecolor/releases/download/v0.0.25/kubecolor_0.0.25_Linux_x86_64.tar.gz

root@k8s-master1:~# cp kubecolor /usr/bin/

pod创建标签是为Deployment控制器对副本数进行管理

创建namespace：

root@k8s-master1:~# kubectl create ns linux
namespace/linux created
root@k8s-master1:~# kubectl create ns python
namespace/python created
root@k8s-master1:~# kubectl label ns linux nsname=linux
namespace/linux labeled
root@k8s-master1:~# kubectl label ns python nsname=python
namespace/python labeled

查看标签

root@k8s-master1:~# kubectl get ns --show-labels
NAME STATUS AGE LABELS
default Active 15h kubernetes.io/metadata.name=default
kube-node-lease Active 15h kubernetes.io/metadata.name=kube-node-lease
kube-public Active 15h kubernetes.io/metadata.name=kube-public
kube-system Active 15h kubernetes.io/metadata.name=kube-system
kubernetes-dashboard Active 15h kubernetes.io/metadata.name=kubernetes-dashboard
linux Active 88s kubernetes.io/metadata.name=linux,nsname=linux
python Active 79s kubernetes.io/metadata.name=python,nsname=python

root@k8s-master1:~/NetWorkPolicy-case/linux-ns1# kubecolor apply -f .
deployment.apps/linux-nginx-deployment created
service/linux-nginx-service created
deployment.apps/linux-tomcat-app1-deployment created
service/linux-tomcat-app1-service created

kubectl apply -f nginx.yaml -f testpod-busybox.yaml -f tomcat.yaml

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubecolor get pods -n linux -o wide

NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES

linux-nginx-deployment-f88c7769d-cqwsd 1/1 Running 0 37s 10.200.118.14 10.4.7.137 <none> <none>

linux-tomcat-app1-deployment-569c85d747-vkq8b 1/1 Running 0 56m 10.200.138.194 10.4.7.138 <none> <none>

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubecolor get pods -n python -o wide

NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES

python-nginx-deployment-5f9bcc696-vldkk 1/1 Running 0 49s 10.200.246.199 10.4.7.139 <none> <none>

python-tomcat-app1-deployment-f956cbb85-ds5wb 1/1 Running 0 7m 10.200.138.195 10.4.7.138 <none> <none>

testpod-busybox-deployment-85f4ccf67b-wrzx5 1/1 Running 0 51m 10.200.246.197 10.4.7.139 <none> <none>

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl exec -it python-tomcat-app1-deployment-f956cbb85-ds5wb -n python bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat#

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat# cd webapps

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat/webapps# ls

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat/webapps# mkdir app

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat/webapps# echo "python web app web page" > app/index.jsp

root@python-tomcat-app1-deployment-f956cbb85-ds5wb:/usr/local/tomcat/webapps# exit

exit

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl exec -it linux-tomcat-app1-deployment-569c85d747-vkq8b -n linux bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

root@linux-tomcat-app1-deployment-569c85d747-vkq8b:/usr/local/tomcat# cd webapps

root@linux-tomcat-app1-deployment-569c85d747-vkq8b:/usr/local/tomcat/webapps# mkdir app

root@linux-tomcat-app1-deployment-569c85d747-vkq8b:/usr/local/tomcat/webapps# echo "linux web app web page" > app/index.jsp

root@linux-tomcat-app1-deployment-569c85d747-vkq8b:/usr/local/tomcat/webapps# exit

exit

default名称空间访问linux 名称空间的tomcat

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl exec -it testpod-busybox-deployment-85f4ccf67b-rmkmn bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

[root@testpod-busybox-deployment-85f4ccf67b-rmkmn /]# curl 10.200.138.194:8080/app/index.jsp

linux web app web page

[root@testpod-busybox-deployment-85f4ccf67b-rmkmn /]#

default名称空间访问python 名称空间的tomcat

[root@testpod-busybox-deployment-85f4ccf67b-rmkmn /]# curl 10.200.138.195:8080/app/index.jsp

python web app web page

linux名称空间下访问python名称空间的pod

[root@net-test-centos-pod1 /]# curl 10.200.138.195:8080/app/index.jsp

python web app web page

[root@net-test-centos-pod1 /]# exit

exit

python名称空间下访问linux名称空间的pod

root@k8s-master1:~# kubectl exec -it net-test-centos-pod1 bash -n python

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

[root@net-test-centos-pod1 /]# curl 10.200.138.194:8080/app/index.jsp

linux web app web page

[root@net-test-centos-pod1 /]#

规则1

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# vim case1-ingress-podSelector.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress #入口方向规则限制

podSelector:

matchLabels:

app: python-tomcat-app1-selector #对匹配到的目的Pod应用以下规则

ingress: #入栈规则，如果指定目标端口就是匹配全部端口和协议，协议TCP, UDP, or SCTP

- from:

- podSelector:

matchLabels:

app: python-nginx-selector #如果存在多个matchLabel条件，如果存在多个matchLabel条件,是and的关系,即要同时满足条件A、条件B、条件X

#project: "python"

#只允许python-nginx-selector访问python-tomcat-app1-selector

在python名称空间访问目标podpython-tomcat-app1-selector

访问不了

从linux名称空间下访问（访问不了）

defualt 名称空间访问访问不了

进入具有指定python-nginx-selector标签的pod 进行访问测试

访问成功

创建一个标签为python-nginx-selector的pod 并访问

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl apply -f testpod-busybox.yaml
deployment.apps/testpod-busybox-deployment created

访问成功

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# vi case1-ingress-podSelector.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress #入口方向规则限制

podSelector:

matchLabels:

app: python-tomcat-app1-selector #对匹配到的目的Pod应用以下规则

ingress: #入栈规则，如果指定目标端口就是匹配全部端口和协议，协议TCP, UDP, or SCTP

- from:

- podSelector:

matchLabels:

#app: python-nginx-selector #如果存在多个matchLabel条件，如果存在多个matchLabel条件,是and的关系,即要同时满足条件A、条件B、条件X

project: "python" #允许同namespace 有python标签的访问python-tomcat-app1-selector这个pod

#只允许python-nginx-selector访问python-tomcat-app1-selector

访问成功

访问失败

查看networkpolicy规则

root@k8s-node2:~# calicoctl get networkpolicy -n python
NAMESPACE NAME
python knp.default.tomcat-access--networkpolicy

root@k8s-node2:~# calicoctl get networkpolicy -n python knp.default.tomcat-access--networkpolicy -o yaml

apiVersion: projectcalico.org/v3

kind: NetworkPolicy

metadata:

creationTimestamp: "2023-03-09T09:17:40Z"

namespace: python

resourceVersion: "31761"

uid: c61f3701-53de-47d3-9484-29762bfaf72f

spec:

ingress:

- action: Allow

destination: {}

source:

selector: projectcalico.org/orchestrator == 'k8s' && project == 'python'

order: 1000

selector: projectcalico.org/orchestrator == 'k8s' && app == 'python-tomcat-app1-selector'

types:

- Ingress

root@k8s-node2:~# iptables-save |grep python
:cali-pri-kns.python - [0:0]
:cali-pri-ksa.python.default - [0:0]
:cali-pro-kns.python - [0:0]
:cali-pro-ksa.python.default - [0:0]
-A cali-fw-calie0bd0343b95 -m comment --comment "cali:EiBQvwGsMOTCY1I6" -j cali-pro-kns.python
-A cali-fw-calie0bd0343b95 -m comment --comment "cali:xV0f35I0MAmJImV3" -j cali-pro-ksa.python.default
-A cali-pri-kns.python -m comment --comment "cali:IwpmNmMD4Wfk3zWm" -j MARK --set-xmark 0x10000/0x10000
-A cali-pri-kns.python -m comment --comment "cali:Ka77pVRcSg0m7yaC" -m mark --mark 0x10000/0x10000 -j RETURN
-A cali-pro-kns.python -m comment --comment "cali:fRot9kFO5ldjxt7s" -j MARK --set-xmark 0x10000/0x10000
-A cali-pro-kns.python -m comment --comment "cali:3MzszMVXH4LZV2U6" -m mark --mark 0x10000/0x10000 -j RETURN
-A cali-tw-calie0bd0343b95 -m comment --comment "cali:VP6WkUvpPThWob-C" -j cali-pri-kns.python
-A cali-tw-calie0bd0343b95 -m comment --comment "cali:_kZSWhMp8o34JS5Z" -j cali-pri-ksa.python.default

2、ingress-以pod加单个端口为限制单位、只允同namespace含有特定标签的源pod访问目标pod的指定端口：

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# vi case2-ingress-podSelector-ns-SinglePort.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress

podSelector:

matchLabels:

app: python-tomcat-app1-selector

ingress:

- from:

- podSelector:

matchLabels:

app: python-nginx-selector #指定访问源的匹配条件,如果存在多个matchLabel条件,是and的关系,即要同时满足条件A、条件B、条件X

#project: "python"

ports: #入栈规则，如果指定目标端口就是匹配全部端口和协议,协议TCP, UDP, or SCTP

- protocol: TCP

port: 8080 #允许通过TCP协议访问目标pod的8080端口,但是其它没有允许的端口将全部禁止访问

#port: 80

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl apply -f case2-ingress-podSelector-ns-SinglePort.yaml

同名称空间访问

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# vi case2-ingress-podSelector-ns-SinglePort.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress

podSelector:

matchLabels:

app: python-tomcat-app1-selector

ingress:

- from:

- podSelector:

matchLabels:

app: python-nginx-selector #指定访问源的匹配条件,如果存在多个matchLabel条件,是and的关系,即要同时满足条件A、条件B、条件X

#project: "python"

ports: #入栈规则，如果指定目标端口就是匹配全部端口和协议,协议TCP, UDP, or SCTP

- protocol: TCP

port: 80 #允许通过TCP协议访问目标pod的8080端口,但是其它没有允许的端口将全部禁止访问

#port: 80

配置文件没有明确声明就是拒绝

重新apply

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl apply -f case2-ingress-podSelector-ns-SinglePort.yaml
networkpolicy.networking.k8s.io/tomcat-access--networkpolicy configured

同名称空间访问

web访问失败

3、ingress-允许同namespace的所有pod访问当前namespace的目标pod多个指定端口：

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# cat case3-ingress-podSelector-ns-MultiPort.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress

podSelector: #目标pod

matchLabels:

app: python-tomcat-app1-selector

ingress:

- from:

- podSelector: #匹配源pod,matchLabels: {}为不限制源pod即允许所有pod,写法等同于resources(不加就是不限制)

matchLabels: {}

ports: #入栈规则，如果指定目标端口就是匹配全部端口和协议，协议TCP, UDP, or SCTP

- protocol: TCP

port: 8080 #允许通过TCP协议访问目标pod的8080端口，但是其它没有允许的端口将全部禁止访问

#port: 80

- protocol: TCP

port: 3306

- protocol: TCP

port: 6379

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl apply -f case3-ingress-podSelector-ns-MultiPort.yaml
networkpolicy.networking.k8s.io/tomcat-access-networkpolicy created

同namespace 所有tomcat pod访问访问成功

-m multiport多个端口匹配

4、ingress-允许同namespace的所有pod访问当前namespace的目标pod所有端口：（不能跨namespace访问）

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# vi case4-ingress-podSelector-ns.yaml

apiVersion: networking.k8s.io/v1

kind: NetworkPolicy

metadata:

namespace: python

spec:

policyTypes:

- Ingress

podSelector: #目标pod

matchLabels: {} #匹配所有目标pod

ingress:

- from:

- podSelector: #匹配源pod,matchLabels: {}为不限制源pod即允许所有pod,写法等同于resources(不加就是不限制)

matchLabels: {}

#ports: #入栈规则，如果指定目标端口就是匹配全部端口和协议，协议TCP, UDP, or SCTP

#- protocol: TCP

# port: {} #允许通过TCP协议访问目标pod的8080端口，但是其它没有允许的端口将全部禁止访问

#port: 80

#- protocol: TCP

# port: 3306

#- protocol: TCP

# port: 6379

root@k8s-master1:~/NetWorkPolicy-case/python-ns2# kubectl apply -f case4-ingress-podSelector-ns.yaml
networkpolicy.networking.k8s.io/tomcat-access--networkpolicy created

新建仓库

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# docker pull eley/slim_java:8

root@k8s-master1:~# docker tag elevy/slim_java:8 www.awen.com/baseimages/elevy/slim_java:8
root@k8s-master1:~# docker push www.awen.com/baseimages/elevy/slim_java:8

测试镜像

root@k8s-master1:~# docker run -it --rm www.awen.com/magedu/zookeeper:v3.4.14

创建pv

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# cat zookeeper-persistentvolume.yaml

---

apiVersion: v1

kind: PersistentVolume

metadata:

spec:

capacity:

storage: 20Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/zookeeper-datadir-1

---

apiVersion: v1

kind: PersistentVolume

metadata:

spec:

capacity:

storage: 20Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/zookeeper-datadir-2

---

apiVersion: v1

kind: PersistentVolume

metadata:

spec:

capacity:

storage: 20Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/zookeeper-datadir-3

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl apply -f zookeeper-persistentvolume.yaml

persistentvolume/zookeeper-datadir-pv-1 created

persistentvolume/zookeeper-datadir-pv-2 created

persistentvolume/zookeeper-datadir-pv-3 created

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl get pv

NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE

zookeeper-datadir-pv-1 20Gi RWO Retain Available 10s

zookeeper-datadir-pv-2 20Gi RWO Retain Available 9s

zookeeper-datadir-pv-3 20Gi RWO Retain Available 9s

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl create ns magedu

namespace/magedu created

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# cat zookeeper-persistentvolumeclaim.yaml

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

namespace: magedu

spec:

accessModes:

- ReadWriteOnce

volumeName: zookeeper-datadir-pv-1

resources:

requests:

storage: 10Gi

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

namespace: magedu

spec:

accessModes:

- ReadWriteOnce

volumeName: zookeeper-datadir-pv-2

resources:

requests:

storage: 10Gi

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

namespace: magedu

spec:

accessModes:

- ReadWriteOnce

volumeName: zookeeper-datadir-pv-3

resources:

requests:

storage: 10Gi

创建pvc

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl apply -f zookeeper-persistentvolumeclaim.yaml
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persistentvolumeclaim/zookeeper-datadir-pvc-3 created

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl get pvc -n magedu

NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE

zookeeper-datadir-pvc-1 Bound zookeeper-datadir-pv-1 20Gi RWO 48s

zookeeper-datadir-pvc-2 Bound zookeeper-datadir-pv-2 20Gi RWO 48s

zookeeper-datadir-pvc-3 Bound zookeeper-datadir-pv-3 20Gi RWO 48s

部署zookeeper

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl apply -f zookeeper.yaml
service/zookeeper created
service/zookeeper1 created
service/zookeeper2 created
service/zookeeper3 created
deployment.apps/zookeeper1 created
deployment.apps/zookeeper2 created
deployment.apps/zookeeper3 created

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl get pod
NAME READY STATUS RESTARTS AGE
net-test1 1/1 Running 0 12h
net-test2 1/1 Running 0 12h
net-test3 1/1 Running 2 (26h ago) 45h

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl exec -it zookeeper1-69774cb565-5rrws bash -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
bash-4.3# /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: follower
bash-4.3# exit
exit
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl exec -it zookeeper2-6f59b4cc44-wspjg bash -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
bash-4.3# /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: follower
bash-4.3# exit
exit
root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper# kubectl exec -it zookeeper3-7f969597c5-qdwch bash -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
bash-4.3# /zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
ZooKeeper remote JMX Port set to 9010
ZooKeeper remote JMX authenticate set to false
ZooKeeper remote JMX ssl set to false
ZooKeeper remote JMX log4j set to true
Using config: /zookeeper/bin/../conf/zoo.cfg
Mode: leader
bash-4.3# exit
exit

部署mysql

下载镜像

root@k8s-master1:~# docker pull mysql:5.6.36

root@k8s-master1:~# docker tag mysql:5.6.36 www.awen.com/magedu/mysql:5.6.36
root@k8s-master1:~# docker push www.awen.com/magedu/mysql:5.6.36

root@k8s-master1:~# docker pull zhangshijie/xtrabackup:1.0

root@k8s-master1:~# docker tag zhangshijie/xtrabackup:1.0 www.awen.com/magedu/xtrabackup:1.0
root@k8s-master1:~# docker push www.awen.com/magedu/xtrabackup:1.0

创建pv

root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-1
root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-2
root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-3
root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-4
root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-5
root@k8s-harbor:/data# mkdir -p /data/k8sdata/magedu/mysql-datadir-6

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# cat mysql-persistentvolume.yaml

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-1

server: 10.4.7.131

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-2

server: 10.4.7.131

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-3

server: 10.4.7.131

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-4

server: 10.4.7.131

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-5

server: 10.4.7.131

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 50Gi

accessModes:

- ReadWriteOnce

nfs:

path: /data/k8sdata/magedu/mysql-datadir-6

server: 10.4.7.131

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# kubectl apply -f mysql-persistentvolume.yaml

persistentvolume/mysql-datadir-1 created

persistentvolume/mysql-datadir-2 created

persistentvolume/mysql-datadir-3 created

persistentvolume/mysql-datadir-4 created

persistentvolume/mysql-datadir-5 created

persistentvolume/mysql-datadir-6 created

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql/pv# kubectl get pv

NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE

mysql-datadir-1 50Gi RWO Retain Available 7m31s

mysql-datadir-2 50Gi RWO Retain Available 7m31s

mysql-datadir-3 50Gi RWO Retain Available 7m31s

mysql-datadir-4 50Gi RWO Retain Available 7m31s

mysql-datadir-5 50Gi RWO Retain Available 7m31s

mysql-datadir-6 50Gi RWO Retain Available 7m31s

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl apply -f mysql-configmap.yaml
configmap/mysql created
root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl apply -f mysql-statefulset.yaml
statefulset.apps/mysql created

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl get pod -n magedu

NAME READY STATUS RESTARTS AGE

mysql-0 2/2 Running 0 49s

mysql-1 2/2 Running 1 (12s ago) 31s

验证

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl exec -it -n magedu mysql-1 bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)

root@mysql-1:/# mysql

Welcome to the MySQL monitor. Commands end with ; or \g.

Your MySQL connection id is 166

Server version: 5.7.36 MySQL Community Server (GPL)

Oracle is a registered trademark of Oracle Corporation and/or its

affiliates. Other names may be trademarks of their respective

owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show salve status\G

ERROR 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'salve status' at line 1

mysql> show slave status\G

*************************** 1. row ***************************

Slave_IO_State: Waiting for master to send event

Master_Host: mysql-0.mysql

Master_User: root

Master_Port: 3306

Connect_Retry: 10

Master_Log_File: mysql-0-bin.000003

Read_Master_Log_Pos: 154

Relay_Log_File: mysql-1-relay-bin.000002

Relay_Log_Pos: 322

Relay_Master_Log_File: mysql-0-bin.000003

Slave_IO_Running: Yes

Slave_SQL_Running: Yes

Replicate_Do_DB:

Replicate_Ignore_DB:

Replicate_Do_Table:

副本扩容

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# vi mysql-statefulset.yaml

apiVersion: apps/v1

kind: StatefulSet

metadata:

namespace: magedu

spec:

selector:

matchLabels:

app: mysql

serviceName: mysql

replicas: 4

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl get pod -n magedu

NAME READY STATUS RESTARTS AGE

mysql-0 2/2 Running 0 10m

mysql-1 2/2 Running 1 (10m ago) 10m

mysql-2 2/2 Running 0 2m55s

mysql-3 2/2 Running 1 (101s ago) 2m1s

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl exec -it -n magedu mysql-2 bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)

root@mysql-2:/# mysql

Welcome to the MySQL monitor. Commands end with ; or \g.

Your MySQL connection id is 289

Server version: 5.7.36 MySQL Community Server (GPL)

Oracle is a registered trademark of Oracle Corporation and/or its

affiliates. Other names may be trademarks of their respective

owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show slave status \G

*************************** 1. row ***************************

Slave_IO_State: Waiting for master to send event

Master_Host: mysql-0.mysql

Master_User: root

Master_Port: 3306

Connect_Retry: 10

Master_Log_File: mysql-0-bin.000003

Read_Master_Log_Pos: 154

Relay_Log_File: mysql-2-relay-bin.000002

Relay_Log_Pos: 322

Relay_Master_Log_File: mysql-0-bin.000003

Slave_IO_Running: Yes

Slave_SQL_Running: Yes

root@k8s-master1:/opt/k8s-data/yaml/magedu/mysql# kubectl exec -it -n magedu mysql-3 bash

kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.

Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)

root@mysql-3:/# mysql

Welcome to the MySQL monitor. Commands end with ; or \g.

Your MySQL connection id is 319

Server version: 5.7.36 MySQL Community Server (GPL)

Oracle is a registered trademark of Oracle Corporation and/or its

affiliates. Other names may be trademarks of their respective

owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> show slave status \G

*************************** 1. row ***************************

Slave_IO_State: Waiting for master to send event

Master_Host: mysql-0.mysql

Master_User: root

Master_Port: 3306

Connect_Retry: 10

Master_Log_File: mysql-0-bin.000003

Read_Master_Log_Pos: 154

Relay_Log_File: mysql-3-relay-bin.000002

Relay_Log_Pos: 322

Relay_Master_Log_File: mysql-0-bin.000003

Slave_IO_Running: Yes

Slave_SQL_Running: Yes

在Kubernetes环境运行java单体服务Jenkins(自己构建镜像或使用官方镜像)、以及实现单Pod中以多容器模式运行基于LNMP的 WordPress(自己构建镜像或使用官方镜像)，数据库使用上一步骤运行在K8S中的MySQL。

root@k8s-master1:/opt/k8s-data/dockerfile/system/centos# cat Dockerfile

#自定义Centos 基础镜像

FROM centos:7.9.2009

MAINTAINER awen tshx163@.com

ADD filebeat-7.12.1-x86_64.rpm /tmp

RUN yum install -y /tmp/filebeat-7.12.1-x86_64.rpm vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop && rm -rf /etc/localtime /tmp/filebeat-7.12.1-x86_64.rpm && ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && useradd nginx -u 2088

构建镜像

root@k8s-master1:/opt/k8s-data/dockerfile/system/centos# cat build-command.sh
#!/bin/bash
docker build -t www.awen.com/baseimages/magedu-centos-base:7.9.2009 .
docker push www.awen.com/baseimages/magedu-centos-base:7.9.2009

#/usr/local/bin/nerdctl build -t harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009 .
#/usr/local/bin/nerdctl push harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/jdk-1.8.212# cat Dockerfile
#JDK Base Image
#FROM harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009
FROM www.awen.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER awen "tshx163@.com"

ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk
ADD profile /etc/profile

ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/jdk-1.8.212# cat build-command.sh
#!/bin/bash
docker build -t www.awen.com/pub-images/jdk-base:v8.212 .
#sleep 1
docker push www.awen.com/pub-images/jdk-base:v8.212

#nerdctl build -t harbor.magedu.net/pub-images/jdk-base:v8.212 .

#nerdctl push harbor.magedu.net/pub-images/jdk-base:v8.212

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/jdk-1.8.212# bash build-command.sh

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/tomcat-base-8.5.43# vi Dockerfile
#Tomcat 8.5.43基础镜像
FROM www.awen.com/pub-images/jdk-base:v8.212

MAINTAINER awen "tshx1991@163.com"

RUN mkdir /apps /data/tomcat/webapps /data/tomcat/logs -pv
ADD apache-tomcat-8.5.43.tar.gz /apps
RUN useradd tomcat -u 2050 && ln -sv /apps/apache-tomcat-8.5.43 /apps/tomcat && chown -R tomcat.tomcat /apps /data -R

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/tomcat-base-8.5.43# vi build-command.sh
#!/bin/bash
docker build -t www.awen.com/pub-images/tomcat-base:v8.5.43 .
#sleep 3
docker push www.awen.com/pub-images/tomcat-base:v8.5.43

#nerdctl build -t harbor.magedu.net/pub-images/tomcat-base:v8.5.43 .

#nerdctl push harbor.magedu.net/pub-images/tomcat-base:v8.5.43

#nerdctl build -t harbor.magedu.net/pub-images/tomcat-base:v8.5.43 .

#nerdctl push harbor.magedu.net/pub-images/tomcat-base:v8.5.43

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/jenkins# cat Dockerfile
#Jenkins Version 2.190.1
#FROM harbor.magedu.net/pub-images/jdk-base:v8.212
FROM www.awen.com/pub-images/jdk-base:v8.212

MAINTAINER awen tshx163@.com

ADD jenkins-2.319.2.war /apps/jenkins/jenkins.war
ADD run_jenkins.sh /usr/bin/

EXPOSE 8080

CMD ["/usr/bin/run_jenkins.sh"]
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/jenkins#

#!/bin/bash
#docker build -t harbor.magedu.net/magedu/jenkins:v2.319.2 .
#echo "镜像制作完成，即将上传至Harbor服务器"
#sleep 1
#docker push harbor.magedu.net/magedu/jenkins:v2.319.2
#echo "镜像上传完成"

echo "即将开始就像构建,请稍等!" && echo 3 && sleep 1 && echo 2 && sleep 1 && echo 1
#nerdctl build -t harbor.linuxarchitect.io/jenkins:v2.319.2 .
docker build -t www.awen.com/magedu/jenkins:v2.319.2 .
if [ $? -eq 0 ];then
echo "即将开始镜像上传,请稍等!" && echo 3 && sleep 1 && echo 2 && sleep 1 && echo 1
docker push www.awen.com/magedu/jenkins:v2.319.2
if [ $? -eq 0 ];then
echo "镜像上传成功!"
else
echo "镜像上传失败"
fi
else
echo "镜像构建失败,请检查构建输出信息!"
fi

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/jenkins# cat run_jenkins.sh
#!/bin/bash
cd /apps/jenkins && java -server -Xms1024m -Xmx1024m -Xss512k -jar jenkins.war --webroot=/apps/jenkins/jenkins-data --httpPort=8080

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/jenkins# bash build-command.sh

测试镜像

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/jenkins# docker run -it --rm -p 8080:8080 www.awen.com/magedu/jenkins:v2.319.2
Running from: /apps/jenkins/jenkins.war
2023-03-11 15:52:01.368+0000 [id=1] INFO org.eclipse.jetty.util.log.Log#initialized: Logging initialized @659ms to org.eclipse.jetty.util.log.JavaUtilLog
2023-03-11 15:52:01.497+0000 [id=1] INFO winstone.Logger#logInternal: Beginning extraction from war file
2023-03-11 15:52:03.692+0000 [id=1] WARNING o.e.j.s.handler.ContextHandler#setContextPath: Empty contextPath
2023-03-11 15:52:03.860+0000 [id=1] INFO org.eclipse.jetty.server.Server#doStart: jetty-9.4.43.v20210629; built: 2021-06-30T11:07:22.254Z; git: 526006ecfa3af7f1a27ef3a288e2bef7ea9dd7e8; jvm 1.8.0_212-b10
2023-03-11 15:52:04.593+0000 [id=1] INFO o.e.j.w.StandardDescriptorProcessor#visitServlet: NO JSP Support for /, did not find org.eclipse.jetty.jsp.JettyJspServlet
2023-03-11 15:52:04.816+0000 [id=1] INFO o.e.j.s.s.DefaultSessionIdManager#doStart: DefaultSessionIdManager workerName=node0
2023-03-11 15:52:04.816+0000 [id=1] INFO o.e.j.s.s.DefaultSessionIdManager#doStart: No SessionScavenger set, using defaults
2023-03-11 15:52:04.827+0000 [id=1] INFO o.e.j.server.session.HouseKeeper#startScavenging: node0 Scavenging every 600000ms
2023-03-11 15:52:06.179+0000 [id=1] INFO hudson.WebAppMain#contextInitialized: Jenkins home directory: /root/.jenkins found at: $user.home/.jenkins
2023-03-11 15:52:06.469+0000 [id=1] INFO o.e.j.s.handler.ContextHandler#doStart: Started w.@6c451c9c{Jenkins v2.319.2,/,file:///apps/jenkins/jenkins-data/,AVAILABLE}{/apps/jenkins/jenkins-data}
2023-03-11 15:52:06.591+0000 [id=1] INFO o.e.j.server.AbstractConnector#doStart: Started ServerConnector@78452606{HTTP/1.1, (http/1.1)}{0.0.0.0:8080}
2023-03-11 15:52:06.591+0000 [id=1] INFO org.eclipse.jetty.server.Server#doStart: Started @5882ms
2023-03-11 15:52:06.594+0000 [id=21] INFO winstone.Logger#logInternal: Winstone Servlet Engine running: controlPort=disabled
2023-03-11 15:52:09.007+0000 [id=28] INFO jenkins.InitReactorRunner$1#onAttained: Started initialization
2023-03-11 15:52:09.060+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained: Listed all plugins
2023-03-11 15:52:11.194+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained: Prepared all plugins
2023-03-11 15:52:11.214+0000 [id=27] INFO jenkins.InitReactorRunner$1#onAttained: Started all plugins
2023-03-11 15:52:11.227+0000 [id=29] INFO jenkins.InitReactorRunner$1#onAttained: Augmented all extensions
2023-03-11 15:52:12.647+0000 [id=29] INFO jenkins.InitReactorRunner$1#onAttained: System config loaded
2023-03-11 15:52:12.649+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained: System config adapted
2023-03-11 15:52:12.650+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained: Loaded all jobs
2023-03-11 15:52:12.651+0000 [id=26] INFO jenkins.InitReactorRunner$1#onAttained: Configuration for all jobs updated
2023-03-11 15:52:12.716+0000 [id=42] INFO hudson.model.AsyncPeriodicWork#lambda$doRun$1: Started Download metadata
2023-03-11 15:52:12.753+0000 [id=42] INFO hudson.util.Retrier#start: Attempt #1 to do the action check updates server
2023-03-11 15:52:13.377+0000 [id=29] INFO jenkins.install.SetupWizard#init:

*************************************************************
*************************************************************
*************************************************************

Jenkins initial setup is required. An admin user has been created and a password generated.
Please use the following password to proceed to installation:

0e8eff38a58b4db0a1588f5d05ce94fd

This may also be found at: /root/.jenkins/secrets/initialAdminPassword

*************************************************************
*************************************************************
*************************************************************

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# vi jenkins-persistentvolume.yaml

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 100Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/jenkins-data

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 100Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/jenkins-root-data # 当前用户家目录

root@k8s-harbor:~# mkdir -p /data/k8sdata/magedu/jenkins-data
root@k8s-harbor:~# mkdir -p /data/k8sdata/magedu/jenkins-root-data

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# kubectl apply -f jenkins-persistentvolume.yaml
persistentvolume/jenkins-datadir-pv created
persistentvolume/jenkins-root-datadir-pv created
root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# kubectl apply -f jenkins-persistentvolumeclaim.yaml
persistentvolumeclaim/jenkins-datadir-pvc created
persistentvolumeclaim/jenkins-root-data-pvc created

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins# kubectl apply -f jenkins.yaml
deployment.apps/magedu-jenkins-deployment created
service/magedu-jenkins-service created

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins# cat jenkins.yaml

kind: Deployment

#apiVersion: extensions/v1beta1

apiVersion: apps/v1

metadata:

labels:

app: magedu-jenkins

namespace: magedu

spec:

replicas: 1

selector:

matchLabels:

app: magedu-jenkins

template:

metadata:

labels:

app: magedu-jenkins

spec:

containers:

- name: magedu-jenkins-container

#image: harbor.magedu.net/magedu/jenkins:v2.319.2

image: www.awen.com/magedu/jenkins:v2.319.2

#imagePullPolicy: IfNotPresent

imagePullPolicy: Always

ports:

- containerPort: 8080

protocol: TCP

volumeMounts:

- mountPath: "/apps/jenkins/jenkins-data/"

- mountPath: "/root/.jenkins"

volumes:

- name: jenkins-datadir-magedu

persistentVolumeClaim:

claimName: jenkins-datadir-pvc

- name: jenkins-root-datadir

persistentVolumeClaim:

claimName: jenkins-root-data-pvc

---

kind: Service

apiVersion: v1

metadata:

labels:

app: magedu-jenkins

namespace: magedu

spec:

type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 8080

nodePort: 38080

selector:

app: magedu-jenkins

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# cat jenkins-persistentvolume.yaml

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 100Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/jenkins-data

---

apiVersion: v1

kind: PersistentVolume

metadata:

namespace: magedu

spec:

capacity:

storage: 100Gi

accessModes:

- ReadWriteOnce

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/jenkins-root-data

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# cat jenkins-persistentvolumeclaim.yaml

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

namespace: magedu

spec:

volumeName: jenkins-datadir-pv

accessModes:

- ReadWriteOnce

resources:

requests:

storage: 80Gi

---

apiVersion: v1

kind: PersistentVolumeClaim

metadata:

namespace: magedu

spec:

volumeName: jenkins-root-datadir-pv

accessModes:

- ReadWriteOnce

resources:

requests:

storage: 80Gi

查看日志找到密码

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# kubectl logs -f -n magedu magedu-jenkins-deployment-868cbfdf47-pktdq

单Pod中以多容器模式运行基于LNMP的 WordPress

nginx-base镜像目录

root@k8s-master1:/opt/k8s-data/yaml/magedu/jenkins/pv# cd /opt/k8s-data/dockerfile/web/pub-images/nginx-base

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# cat Dockerfile
#Nginx Base Image
FROM www.awen.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER awen

RUN yum install -y vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure && make && make install && ln -sv /usr/local/nginx/sbin/nginx /usr/sbin/nginx &&rm -rf /usr/local/src/nginx-1.22.0.tar.gz
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# cat build-command.sh
#!/bin/bash
docker build -t www.awen.com/pub-images/nginx-base:v1.22.0 .
#sleep 1
docker push www.awen.com/pub-images/nginx-base:v1.22.0

#nerdctl build -t harbor.linuxarchitect.io/pub-images/nginx-base:v1.22.0 .

#nerdctl push harbor.linuxarchitect.io/pub-images/nginx-base:v1.22.0

构建镜像

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base# bash build-command.sh

nginx-base-wordpress镜像构建目录

root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# cat Dockerfile
#Nginx Base Image
FROM www.awen.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER zhangshijie@magedu.net

RUN yum install -y vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx && make && make install && ln -sv /apps/nginx/sbin/nginx /usr/sbin/nginx &&rm -rf /usr/local/src/nginx-1.22.0.tar.gz
root@k8s-master1:/opt/k8s-data/dockerfile/web/pub-images/nginx-base-wordpress# cat build-command.sh
#!/bin/bash
docker build -t www.awen.com/pub-images/nginx-base-wordpress:v1.22.0 .
#sleep 1
docker push www.awen.com/pub-images/nginx-base-wordpress:v1.22.0

#nerdctl build -t harbor.magedu.net/pub-images/nginx-base-wordpress:v1.22.0 .

#nerdctl push harbor.magedu.net/pub-images/nginx-base-wordpress:v1.22.0

wordpress构建构建目录

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat Dockerfile
#FROM harbor.magedu.local/pub-images/nginx-base-wordpress:v1.20.2
FROM www.awen.com/pub-images/nginx-base-wordpress:v1.22.0

ADD nginx.conf /apps/nginx/conf/nginx.conf
ADD run_nginx.sh /apps/nginx/sbin/run_nginx.sh
RUN mkdir -pv /home/nginx/wordpress
RUN chown nginx.nginx /home/nginx/wordpress/ -R

EXPOSE 80 443

CMD ["/apps/nginx/sbin/run_nginx.sh"]

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t www.awen.com/magedu/wordpress-nginx:${TAG} .
#nerdctl build -t harbor.magedu.net/magedu/wordpress-nginx:${TAG} .
echo "镜像制作完成，即将上传至Harbor服务器"
sleep 1
#nerdctl push harbor.magedu.net/magedu/wordpress-nginx:${TAG}
docker push www.awen.com/magedu/wordpress-nginx:${TAG}
echo "镜像上传完成"
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# cat run_nginx.sh
#!/bin/bash
#echo "nameserver 10.20.254.254" > /etc/resolv.conf
#chown nginx.nginx /home/nginx/wordpress/ -R
/apps/nginx/sbin/nginx
tail -f /etc/hosts

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/nginx# bash build-command.sh v1

php

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat Dockerfile
#PHP Base Image
#FROM harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009
FROM www.awen.com/baseimages/magedu-centos-base:7.9.2009

MAINTAINER awen@163.com

RUN yum install -y https://mirrors.tuna.tsinghua.edu.cn/remi/enterprise/remi-release-7.rpm && yum install php56-php-fpm php56-php-mysql -y
ADD www.conf /opt/remi/php56/root/etc/php-fpm.d/www.conf
#RUN useradd nginx -u 2019
ADD run_php.sh /usr/local/bin/run_php.sh
EXPOSE 9000

CMD ["/usr/local/bin/run_php.sh"]
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat build-command.sh
#!/bin/bash
TAG=$1
docker build -t www.awen.com/magedu/wordpress-php-5.6:${TAG} .
#nerdctl build -t harbor.magedu.net/magedu/wordpress-php-5.6:${TAG} .
echo "镜像制作完成，即将上传至Harbor服务器"
sleep 1
#nerdctl push harbor.magedu.net/magedu/wordpress-php-5.6:${TAG}
docker push www.awen.com/magedu/wordpress-php-5.6:${TAG}
echo "镜像上传完成"
root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# cat run_php.sh
#!/bin/bash
#echo "nameserver 10.20.254.254" > /etc/resolv.conf

/opt/remi/php56/root/usr/sbin/php-fpm
#/opt/remi/php56/root/usr/sbin/php-fpm --nodaemonize
tail -f /etc/hosts

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# bash build-command.sh v1

部署wordpress

root@k8s-harbor:~# mkdir -p /data/k8sdata/magedu/wordpress

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/wordpress/php# bash build-command.sh v1

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl apply -f wordpress.yaml
deployment.apps/wordpress-app-deployment created
service/wordpress-app-spec created

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl get pod -n magedu

NAME READY STATUS RESTARTS AGE

wordpress-app-deployment-7485886d54-blnzh 2/2 Running 0 49s

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# cat wordpress.yaml

kind: Deployment

#apiVersion: extensions/v1beta1

apiVersion: apps/v1

metadata:

labels:

app: wordpress-app

namespace: magedu

spec:

replicas: 1

selector:

matchLabels:

app: wordpress-app

template:

metadata:

labels:

app: wordpress-app

spec:

containers:

- name: wordpress-app-nginx

image: www.awen.com/magedu/wordpress-nginx:v1

imagePullPolicy: Always

ports:

- containerPort: 80

protocol: TCP

- containerPort: 443

protocol: TCP

volumeMounts:

- name: wordpress

mountPath: /home/nginx/wordpress

readOnly: false

- name: wordpress-app-php

image: www.awen.com/magedu/wordpress-php-5.6:v1

#image: harbor.magedu.net/magedu/php:5.6.40-fpm

#imagePullPolicy: IfNotPresent

imagePullPolicy: Always

ports:

- containerPort: 9000

protocol: TCP

volumeMounts:

- name: wordpress

mountPath: /home/nginx/wordpress

readOnly: false

volumes:

- name: wordpress

nfs:

server: 10.4.7.131

path: /data/k8sdata/magedu/wordpress

---

kind: Service

apiVersion: v1

metadata:

labels:

app: wordpress-app

namespace: magedu

spec:

type: NodePort

ports:

- name: http

port: 80

protocol: TCP

targetPort: 80

nodePort: 30031

- name: https

port: 443

protocol: TCP

targetPort: 443

nodePort: 30033

selector:

app: wordpress-app

流量通过LB 转发给service 转发给pod

root@k8s-ha:~# vi /etc/haproxy/haproxy.cfg

listen magedu-woedpress-80
bind 10.4.7.51:80
mode tcp
#balance leastconn
server 10.4.7.134 10.4.7.134:30031 check inter 2000 fall 3 rise 5
server 10.4.7.135 10.4.7.135:30031 check inter 2000 fall 3 rise 5
server 10.4.7.136 10.4.7.136:30031 check inter 2000 fall 3 rise 5

测试页面

root@k8s-harbor:/data/k8sdata/magedu/wordpress# cat index.php
<?php
phpinfo();
?>

宿主机添加hosts解析记录

10.4.7.51 www.mysite.com

部署wordpress站点文件

root@k8s-harbor:/data/k8sdata/magedu/wordpress# tar xf wordpress-5.0.16-zh_CN.tar.gz

部署wordpress站点文件把资源文件移动到wordpress部署的根目录

root@k8s-harbor:/data/k8sdata/magedu/wordpress# mv wordpress/* .

root@k8s-harbor:/data/k8sdata/magedu/wordpress# mv wordpress wordpress-5.0.16-zh_CN.tar.gz /tmp

查看nginx和php的uid

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl exec -it -n magedu wordpress-app-deployment-796d7bbdf-w85zc bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "wordpress-app-nginx" out of: wordpress-app-nginx, wordpress-app-php
[root@wordpress-app-deployment-796d7bbdf-w85zc /]# id nginx
uid=2088(nginx) gid=2088(nginx) groups=2088(nginx)

修改NFS服务器wordpress数据库目录权限

root@k8s-harbor:/data/k8sdata/magedu# chown 2088.2088 wordpress/ -R

创建数据库

root@k8s-master1:/opt/k8s-data/yaml/magedu/wordpress# kubectl exec -it mysql-0 sh -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
Defaulted container "mysql" out of: mysql, xtrabackup, init-mysql (init), clone-mysql (init)
# mysql
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 17033
Server version: 5.7.36-log MySQL Community Server (GPL)

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> CREATE DATABASE wordpress;
Query OK, 1 row affected (0.03 sec)

mysql> GRANT ALL PRIVILEGES ON wordpress.* TO "wordpress"@"%" IDENTIFIED BY "wordpress";
Query OK, 0 rows affected, 1 warning (0.06 sec)

配置文件

root@k8s-harbor:/data/k8sdata/magedu/wordpress# cat wp-config.php

基于Operator在Kubernetes环境部署prometheus监控环境(prometheus-server、cAdvisor、grafana、node-exporter等)

root@k8s-master1:~# git clone -b release-0.11 https://github.com/prometheus-operator/kube-prometheus.git

Cloning into 'kube-prometheus'...

remote: Enumerating objects: 17669, done.

remote: Counting objects: 100% (218/218), done.

remote: Compressing objects: 100% (114/114), done.

remote: Total 17669 (delta 150), reused 140 (delta 94), pack-reused 17451

Receiving objects: 100% (17669/17669), 9.28 MiB | 27.00 KiB/s, done.

Resolving deltas: 100% (11652/11652), done.

root@k8s-master1:~# docker tag bitnami/kube-state-metrics:2.5.0 www.awen.com/baseimages/kube-state-metrics:2.5.0

root@k8s-master1:~# docker push www.awen.com/baseimages/kube-state-metrics:2.5.0

root@k8s-master1:~# docker tag willdockerhub/prometheus-adapter:v0.9.1 www.awen.com/baseimages/prometheus-adapter:v0.9.1

root@k8s-master1:~# docker push www.awen.com/baseimages/prometheus-adapter:v0.9.1

修改yaml文件中的镜像地址

root@k8s-master1:~/kube-prometheus# cat manifests/kubeStateMetrics-deployment.yaml | grep image

image: k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.5.0

image: quay.io/brancz/kube-rbac-proxy:v0.12.0

root@k8s-master1:~/kube-prometheus# cat manifests/prometheusAdapter-deployment.yaml | grep image

image: k8s.gcr.io/prometheus-adapter/prometheus-adapter:v0.9.1

root@k8s-master1:~/kube-prometheus# cat manifests/prometheusAdapter-deployment.yaml | grep image

image: www.awen.com/baseimages/prometheus-adapter:v0.9.1

root@k8s-master1:~/kube-prometheus# cat manifests/kubeStateMetrics-deployment.yaml | grep image

image: www.awen.com/baseimages/kube-state-metrics:v2.5.0

image: quay.io/brancz/kube-rbac-proxy:v0.12.0

root@k8s-master1:~/kube-prometheus# kubectl create -f manifests/setup/

customresourcedefinition.apiextensions.k8s.io/alertmanagerconfigs.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/alertmanagers.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/podmonitors.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/probes.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/prometheuses.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/prometheusrules.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/servicemonitors.monitoring.coreos.com created

customresourcedefinition.apiextensions.k8s.io/thanosrulers.monitoring.coreos.com created

namespace/monitoring created

root@k8s-master1:~/kube-prometheus# kubectl apply -f manifests/

alertmanager.monitoring.coreos.com/main created

networkpolicy.networking.k8s.io/alertmanager-main created

poddisruptionbudget.policy/alertmanager-main created

prometheusrule.monitoring.coreos.com/alertmanager-main-rules created

secret/alertmanager-main created

service/alertmanager-main created

serviceaccount/alertmanager-main created

servicemonitor.monitoring.coreos.com/alertmanager-main created

clusterrole.rbac.authorization.k8s.io/blackbox-exporter created

clusterrolebinding.rbac.authorization.k8s.io/blackbox-exporter created

configmap/blackbox-exporter-configuration created

deployment.apps/blackbox-exporter created

networkpolicy.networking.k8s.io/blackbox-exporter created

service/blackbox-exporter created

serviceaccount/blackbox-exporter created

servicemonitor.monitoring.coreos.com/blackbox-exporter created

secret/grafana-config created

secret/grafana-datasources created

configmap/grafana-dashboard-alertmanager-overview created

configmap/grafana-dashboard-apiserver created

configmap/grafana-dashboard-cluster-total created

configmap/grafana-dashboard-controller-manager created

configmap/grafana-dashboard-grafana-overview created

configmap/grafana-dashboard-k8s-resources-cluster created

configmap/grafana-dashboard-k8s-resources-namespace created

configmap/grafana-dashboard-k8s-resources-node created

configmap/grafana-dashboard-k8s-resources-pod created

configmap/grafana-dashboard-k8s-resources-workload created

configmap/grafana-dashboard-k8s-resources-workloads-namespace created

configmap/grafana-dashboard-kubelet created

configmap/grafana-dashboard-namespace-by-pod created

configmap/grafana-dashboard-namespace-by-workload created

configmap/grafana-dashboard-node-cluster-rsrc-use created

configmap/grafana-dashboard-node-rsrc-use created

configmap/grafana-dashboard-nodes created

configmap/grafana-dashboard-persistentvolumesusage created

configmap/grafana-dashboard-pod-total created

configmap/grafana-dashboard-prometheus-remote-write created

configmap/grafana-dashboard-prometheus created

configmap/grafana-dashboard-proxy created

configmap/grafana-dashboard-scheduler created

configmap/grafana-dashboard-workload-total created

configmap/grafana-dashboards created

deployment.apps/grafana created

networkpolicy.networking.k8s.io/grafana created

prometheusrule.monitoring.coreos.com/grafana-rules created

service/grafana created

serviceaccount/grafana created

servicemonitor.monitoring.coreos.com/grafana created

prometheusrule.monitoring.coreos.com/kube-prometheus-rules created

clusterrole.rbac.authorization.k8s.io/kube-state-metrics created

clusterrolebinding.rbac.authorization.k8s.io/kube-state-metrics created

deployment.apps/kube-state-metrics created

networkpolicy.networking.k8s.io/kube-state-metrics created

prometheusrule.monitoring.coreos.com/kube-state-metrics-rules created

service/kube-state-metrics created

serviceaccount/kube-state-metrics created

servicemonitor.monitoring.coreos.com/kube-state-metrics created

prometheusrule.monitoring.coreos.com/kubernetes-monitoring-rules created

servicemonitor.monitoring.coreos.com/kube-apiserver created

servicemonitor.monitoring.coreos.com/coredns created

servicemonitor.monitoring.coreos.com/kube-controller-manager created

servicemonitor.monitoring.coreos.com/kube-scheduler created

servicemonitor.monitoring.coreos.com/kubelet created

clusterrole.rbac.authorization.k8s.io/node-exporter created

clusterrolebinding.rbac.authorization.k8s.io/node-exporter created

daemonset.apps/node-exporter created

networkpolicy.networking.k8s.io/node-exporter created

prometheusrule.monitoring.coreos.com/node-exporter-rules created

service/node-exporter created

serviceaccount/node-exporter created

servicemonitor.monitoring.coreos.com/node-exporter created

clusterrole.rbac.authorization.k8s.io/prometheus-k8s created

clusterrolebinding.rbac.authorization.k8s.io/prometheus-k8s created

networkpolicy.networking.k8s.io/prometheus-k8s created

poddisruptionbudget.policy/prometheus-k8s created

prometheus.monitoring.coreos.com/k8s created

prometheusrule.monitoring.coreos.com/prometheus-k8s-prometheus-rules created

rolebinding.rbac.authorization.k8s.io/prometheus-k8s-config created

rolebinding.rbac.authorization.k8s.io/prometheus-k8s created

role.rbac.authorization.k8s.io/prometheus-k8s-config created

role.rbac.authorization.k8s.io/prometheus-k8s created

service/prometheus-k8s created

serviceaccount/prometheus-k8s created

servicemonitor.monitoring.coreos.com/prometheus-k8s created

apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io configured

clusterrole.rbac.authorization.k8s.io/prometheus-adapter created

clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader configured

clusterrolebinding.rbac.authorization.k8s.io/prometheus-adapter created

clusterrolebinding.rbac.authorization.k8s.io/resource-metrics:system:auth-delegator created

clusterrole.rbac.authorization.k8s.io/resource-metrics-server-resources created

configmap/adapter-config created

deployment.apps/prometheus-adapter created

networkpolicy.networking.k8s.io/prometheus-adapter created

poddisruptionbudget.policy/prometheus-adapter created

rolebinding.rbac.authorization.k8s.io/resource-metrics-auth-reader created

service/prometheus-adapter created

serviceaccount/prometheus-adapter created

servicemonitor.monitoring.coreos.com/prometheus-adapter created

clusterrole.rbac.authorization.k8s.io/prometheus-operator created

clusterrolebinding.rbac.authorization.k8s.io/prometheus-operator created

deployment.apps/prometheus-operator created

networkpolicy.networking.k8s.io/prometheus-operator created

prometheusrule.monitoring.coreos.com/prometheus-operator-rules created

service/prometheus-operator created

serviceaccount/prometheus-operator created

servicemonitor.monitoring.coreos.com/prometheus-operator created

networkPolicy

root@k8s-master1:~/kube-prometheus# ll manifests/*network*

-rw-r--r-- 1 root root 977 Mar 12 13:14 manifests/alertmanager-networkPolicy.yaml

-rw-r--r-- 1 root root 722 Mar 12 13:14 manifests/blackboxExporter-networkPolicy.yaml

-rw-r--r-- 1 root root 651 Mar 12 13:14 manifests/grafana-networkPolicy.yaml

-rw-r--r-- 1 root root 723 Mar 12 13:14 manifests/kubeStateMetrics-networkPolicy.yaml

-rw-r--r-- 1 root root 671 Mar 12 13:14 manifests/nodeExporter-networkPolicy.yaml

-rw-r--r-- 1 root root 564 Mar 12 13:14 manifests/prometheusAdapter-networkPolicy.yaml

-rw-r--r-- 1 root root 922 Mar 12 13:14 manifests/prometheus-networkPolicy.yaml

-rw-r--r-- 1 root root 694 Mar 12 13:14 manifests/prometheusOperator-networkPolicy.yaml

root@k8s-master1:~/kube-prometheus# vi manifests/grafana-service.yaml

apiVersion: v1

kind: Service

metadata:

labels:

app.kubernetes.io/component: grafana

app.kubernetes.io/name: grafana

app.kubernetes.io/part-of: kube-prometheus

app.kubernetes.io/version: 8.5.5

namespace: monitoring

spec:

type: NodePort

ports:

- name: http

port: 3000

targetPort: http

nodePort: 33000

selector:

app.kubernetes.io/component: grafana

app.kubernetes.io/name: grafana

app.kubernetes.io/part-of: kube-prometheus

root@k8s-master1:~/kube-prometheus# kubectl apply -f manifests/grafana-service.yaml

service/grafana configured

root@k8s-master1:~/kube-prometheus# kubectl delete -f networkPolicy/

root@k8s-master1:~/prometheus-case-files# cat case1-daemonset-deploy-cadvisor.yaml

apiVersion: apps/v1

kind: DaemonSet

metadata:

namespace: monitoring

spec:

selector:

matchLabels:

app: cAdvisor

template:

metadata:

labels:

app: cAdvisor

spec:

tolerations: #污点容忍,忽略master的NoSchedule

- effect: NoSchedule

key: node-role.kubernetes.io/master

hostNetwork: true

restartPolicy: Always # 重启策略

containers:

- name: cadvisor

image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/cadvisor-amd64:v0.45.0

imagePullPolicy: IfNotPresent # 镜像策略

ports:

- containerPort: 8080

volumeMounts:

- name: root

mountPath: /rootfs

- name: run

mountPath: /var/run

- name: sys

mountPath: /sys

- name: docker

mountPath: /var/lib/containerd

volumes:

- name: root

hostPath:

path: /

- name: run

hostPath:

path: /var/run

- name: sys

hostPath:

path: /sys

- name: docker

hostPath:

path: /var/lib/containerd

root@k8s-master1:~/prometheus-case-files# kubectl create ns monitoring

namespace/monitoring created

root@k8s-master1:~/prometheus-case-files# kubectl apply -f case1-daemonset-deploy-cadvisor.yaml

daemonset.apps/cadvisor created

root@k8s-master1:~/prometheus-case-files# kubectl get pod -n monitoring -o wide

NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES

cadvisor-8q9dw 1/1 Running 0 9m54s 10.4.7.137 10.4.7.137 <none> <none>

cadvisor-b9ppl 1/1 Running 0 9m53s 10.4.7.138 10.4.7.138 <none> <none>

cadvisor-jqtxl 1/1 Running 0 9m53s 10.4.7.139 10.4.7.139 <none> <none>

cadvisor-jqzxl 1/1 Running 0 9m53s 10.4.7.135 10.4.7.135 <none> <none>

cadvisor-kvr9g 1/1 Running 0 9m53s 10.4.7.134 10.4.7.134 <none> <none>

cadvisor-p8np6 1/1 Running 0 9m53s 10.4.7.136 10.4.7.136 <none> <none>

手动在K8S部署prometheus server并监控K8S集群：

部署cadvisor:

pod的指标数据

手动在K8S部署prometheus server并监控K8S集群：

部署node_exporter:

root@k8s-master1:~/prometheus-case-files# cat case2-daemonset-deploy-node-exporter.yaml

apiVersion: apps/v1

kind: DaemonSet

metadata:

namespace: monitoring

labels:

k8s-app: node-exporter

spec:

selector:

matchLabels:

k8s-app: node-exporter

template:

metadata:

labels:

k8s-app: node-exporter

spec:

tolerations:

- effect: NoSchedule

key: node-role.kubernetes.io/master

containers:

- image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/node-exporter:v1.5.0

imagePullPolicy: IfNotPresent

ports:

- containerPort: 9100

hostPort: 9100

protocol: TCP

volumeMounts:

- mountPath: /host/proc

- mountPath: /host/sys

- mountPath: /host

args:

- --path.procfs=/host/proc

- --path.sysfs=/host/sys

- --path.rootfs=/host

volumes:

- name: proc

hostPath:

path: /proc

- name: sys

hostPath:

path: /sys

- name: rootfs

hostPath:

path: /

hostNetwork: true

hostPID: true

---

apiVersion: v1

kind: Service

metadata:

annotations:

prometheus.io/scrape: "true"

labels:

k8s-app: node-exporter

namespace: monitoring

spec:

type: NodePort

ports:

- name: http

port: 9100

nodePort: 39100

protocol: TCP

selector:

k8s-app: node-exporter

root@k8s-master1:~/prometheus-case-files# kubectl apply -f case2-daemonset-deploy-node-exporter.yaml
daemonset.apps/node-exporter created
service/node-exporter created

部署prometheus server:

root@k8s-harbor:~# apt install nfs-server

root@k8s-harbor:~# apt-get install nfs-kernel-server

root@k8s-harbor:~# mkdir -p /data/k8sdata/prometheusdata
root@k8s-harbor:~# chown 65534.65534 /data/k8sdata/prometheusdata -R

root@k8s-harbor:~# vi /etc/exports

root@k8s-harbor:~# systemctl restart nfs-server
/data/k8sdata/ 10.4.7.*(rw,sync,no_root_squash,no_subtree_check)

创建监控账号赋予权限动态发现k8s的pod

root@k8s-master1:~/prometheus-case-files# kubectl create serviceaccount monitor -n monitoring
serviceaccount/monitor created

账号授权

root@k8s-master1:~/prometheus-case-files# kubectl create clusterrolebinding monitor-clusterrolebindong -n monitoring --clusterrole=cluster-admin --serviceaccount=monitoring:monitor
clusterrolebinding.rbac.authorization.k8s.io/monitor-clusterrolebindong created