k8s之devops 的实现

设置持久数据目录nfs参考： https://www.cnblogs.com/rdchenxi/p/17113820.html

创建pv与pvc

[root@master devops]# cat jenkins-pv.yaml  jenkins-pvc.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: jenkins-k8s-pv
spec: 
  capacity:
    storage: 10Gi
  accessModes:
  - ReadWriteMany
  nfs:
    server: 192.168.40.50
    path: /data/kubernetes/pv1
 
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: jenkins-k8s-pvc
  namespace: jenkins-k8s
spec:
  resources:
    requests:
      storage: 10Gi
  accessModes:
  - ReadWriteMany
  #volumeName: jenkins-k8s-pv




 kubectl apply -f jenkins-pv.yaml
kubectl apply -f jenkins-pvc.yaml
[root@master devops]# kubectl get pv
NAME             CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
jenkins-k8s-pv   10Gi       RWX            Retain           Available                                   9m29s

　　在k8s里面创建jenkins 账户授权

[root@master devops]# kubectl create sa jenkins-k8s-sa -n jenkins-k8s 
serviceaccount/jenkins-k8s-sa created
[root@master devops]# kubectl create clusterrolebinding jenkins-k8s-sa-cluster -n jenkins-k8s  --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa
clusterrolebinding.rbac.authorization.k8s.io/jenkins-k8s-sa-cluster created

　　创建jenkins 的dep

[root@master devops]# cat jenkins-dep.yaml 
kind: Deployment
apiVersion: apps/v1
metadata:
  name: jenkins
  namespace: jenkins-k8s
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccount: jenkins-k8s-sa
      containers:
      - name: jenkins
        image:  rdchenxi/jenkins:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          name: web
          protocol: TCP
        - containerPort: 50000
          name: agent
          protocol: TCP
        livenessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        readinessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        volumeMounts:
        - name: jenkins-volume
          subPath: jenkins-home
          mountPath: /var/jenkins_home
      volumes:
      - name: jenkins-volume
        persistentVolumeClaim:
          claimName: jenkins-k8s-pvc

[root@master devops]# kubectl apply -f jenkins-dep.yaml 
deployment.apps/jenkins created

　　jenkins 的svc 实现

vim jenkins-svc.yaml


apiVersion: v1
kind: Service
metadata:
  name: jenkins-service
  namespace: jenkins-k8s
  labels:
    app: jenkins
spec:
  selector:
    app: jenkins
  type: NodePort
  ports:
  - name: web
    port: 8080
    targetPort: web
    nodePort: 30002
  - name: agent
    port: 50000
    targetPort: agent

[root@master devops]# kubectl apply -f jenkins-svc.yaml 
service/jenkins-service created
[root@master devops]# kubectl describe svc -n jenkins-k8s jenkins-service 
Name:                     jenkins-service
Namespace:                jenkins-k8s
Labels:                   app=jenkins
Annotations:              <none>
Selector:                 app=jenkins
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       10.108.234.89
IPs:                      10.108.234.89
Port:                     web  8080/TCP
TargetPort:               web/TCP
NodePort:                 web  30002/TCP
Endpoints:                10.244.84.139:8080
Port:                     agent  50000/TCP
TargetPort:               agent/TCP
NodePort:                 agent  31126/TCP
Endpoints:                10.244.84.139:50000
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>

　　访问地址：http://192.168.40.50:30002/

安装全部k8s 插件

配置jenkins 连接k8s

 名字

地址

 所在名称空间

 jenkins 集群里的地址：http://jenkins-service.jenkins-k8s.svc.cluster.local:8080

pod 模版

 

容器的配置 

 卷

工具

 

模版pod 的用户ID

 

　　系统配置  http://jenkins-service.jenkins-k8s.svc.cluster.local:8080/

 

 

 

 sa

配置docker 的凭据

 

 创建3个名称空间对应3个

kubectl create ns production
kubectl create ns qatest
kubectl create ns devlopment

　　构建任务

node('test-devops') {
    stage('Clone') {
        echo "1.Clone Stage"
        git url: "https://github.com/ruidongchenxi/jenkins-sample.git"
        script {
            build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
        }
    }
    stage('Test') {
      echo "2.Test Stage"

    }
    stage('Build') {
        echo "3.Build Docker Image Stage"
        sh "docker build -t rdchenxi/jenkins-demo:${build_tag} ."
    }
    stage('Push') {
        echo "4.Push Docker Image Stage"
        withCredentials([usernamePassword(credentialsId: 'dockerhub', passwordVariable: 'dockerHubPassword', usernameVariable: 'dockerHubUser')]) {
            sh "docker login -u ${dockerHubUser} -p ${dockerHubPassword}"
            sh "docker push rdchenxi/jenkins-demo:${build_tag}"
        }
    }
    stage('Deploy to dev') {
        echo "5. Deploy DEV"
		sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-dev.yaml"
        sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-dev.yaml"
//        sh "bash running-devlopment.sh"
        sh "kubectl apply -f k8s-dev.yaml  --validate=false"
	}	
	stage('Promote to qa') {	
		def userInput = input(
            id: 'userInput',

            message: 'Promote to qa?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-qa.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-qa.yaml"
//            sh "bash running-qa.sh"
            sh "kubectl apply -f k8s-qa.yaml --validate=false"
            sh "sleep 6"
            sh "kubectl get pods -n qatest"
        } else {
            //exit
        }
    }
	stage('Promote to pro') {	
		def userInput = input(

            id: 'userInput',
            message: 'Promote to pro?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-prod.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-prod.yaml"
//            sh "bash running-production.sh"
            sh "cat k8s-prod.yaml"
            sh "kubectl apply -f k8s-prod.yaml --record --validate=false"
        }
    }
}

[root@master ~]# kubectl get po -n production 
NAME                            READY   STATUS    RESTARTS   AGE
jenkins-demo-6669c88569-4685w   1/1     Running   0          57m

　　配置私有镜像仓库密码

node  docker相关设置，否则无法推送镜像

vim  /etc/docker/daemon.json
{
  "registry-mirrors": ["https://g2aogmw8.mirror.aliyuncs.com"],
  "insecure-registries": ["192.168.40.42","harbor"]
}


[root@node-1 ~]# vim /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.40.50 master
192.168.40.51 node
192.168.40.42 Harbor 192.168.40.42
重启docker
 systemctl restart docker.service 

　　

node 节点containerd 配置从私有仓库拉镜像

mkdir -p /etc/containerd/certs.d/192.168.40.42
vim /etc/containerd/certs.d/192.168.40.42/hosts.toml
server = "https://192.168.40.42"

[host."http://192.168.40.42"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
重启
systemctl restart containerd
systemctl restart kubelet

　　

　　在私有仓库构建镜像库

 

流水线项目

node('test-devops') {
    stage('Clone') {
        echo "1.Clone Stage"
        git url: "https://github.com/ruidongchenxi/jenkins-sample.git"
        script {
            build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
        }
    }
    stage('Test') {
      echo "2.Test Stage"

    }
    stage('Build') {
        echo "3.Build Docker Image Stage"
        sh "docker build -t 192.168.40.42/jenkins-demo/jenkins-demo:${build_tag} ."
    }
    stage('Push') {
        echo "4.Push Docker Image Stage"
        withCredentials([usernamePassword(credentialsId: 'dockerharbor', passwordVariable: 'dockerHubPassword', usernameVariable: 'dockerHubUser')]) {
            sh "docker login 192.168.40.42 -u ${dockerHubUser} -p ${dockerHubPassword}"
            sh "docker push 192.168.40.42/jenkins-demo/jenkins-demo:${build_tag}"
        }
    }
    stage('Deploy to dev') {
        echo "5. Deploy DEV"
		sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-dev-harbor.yaml"
        sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-dev-harbor.yaml"
//        sh "bash running-devlopment.sh"
        sh "kubectl apply -f k8s-dev-harbor.yaml  --validate=false"
	}	
	stage('Promote to qa') {	
		def userInput = input(
            id: 'userInput',

            message: 'Promote to qa?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-qa-harbor.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-qa-harbor.yaml"
//            sh "bash running-qa.sh"
            sh "kubectl apply -f k8s-qa-harbor.yaml --validate=false"
            sh "sleep 6"
            sh "kubectl get pods -n qatest"
        } else {
            //exit
        }
    }
	stage('Promote to pro') {	
		def userInput = input(

            id: 'userInput',
            message: 'Promote to pro?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-prod-harbor.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-prod-harbor.yaml"
//            sh "bash running-production.sh"
            sh "cat k8s-prod-harbor.yaml"
            sh "kubectl apply -f k8s-prod-harbor.yaml --record --validate=false"
        }
    }
}