学习K8S之路.6--- 在K8S中部署Jenkins，并使用Jenkins打包jar包

一：部署jenkins

jenkins官网：https://jenkins.io/download/
jenkins镜像：https://hub.docker.com/r/jenkins/jenkins

1：在运维主机(192.168.6.96)上下载官网稳定版镜像

[root@k8s-6-96 ~]# docker pull jenkins/jenkins:2.190.3
[root@k8s-6-96 ~]# docker images | grep jenkins
jenkins/jenkins                    2.190.3                    22b8b9a84dbe        6 months ago        568MB
[root@k8s-6-96 ~]# docker tag 22b8b9a84dbe harbor.auth.com/public/jenkins:v2.190.3
[root@k8s-6-96 ~]# docker push harbor.auth.com/public/jenkins:v2.190.3

2：制作自定义镜像

　　2.1：生成ssh秘钥对，用于拉取git私有仓库代码，需要使用ssh密钥

[root@k8s-6-96 ~]# ssh-keygen -t rsa -b 2048 -C "117586xxx@qq.com" -N "" -f /root/.ssh/id_rsa
// 执行完成会在/root/.ssh/ 目录下生成一对密钥

　　2.2：准备get-docker.sh文件，用于在jenkins容器中使用docker客户端，执行 build

[root@k8s-6-96 ~]# curl -fsSL get.docker.com -o get-docker.sh
[root@k8s-6-96 ~]# chmod +x get-docker.sh

　　2.3：准备config.json文件，用于把docker镜像push到镜像仓库

[root@k8s-6-96 ~]# cat  /root/.docker/config.json
{
    "auths": {
        "harbor.od.com": {
            "auth": "YWRtaW46SGFyYm9yMTIzNDU="
        }
    },
    "HttpHeaders": {
        "User-Agent": "Docker-Client/19.03.4 (linux)"
    }

　　2.4：创建目录并准备Dockerfile

[root@k8s-6-96 ~]# cd /data/dockerfile/
[root@k8s-6-96 dockerfile]# mkdir jenkins
[root@k8s-6-96 dockerfile]# cd jenkins

[root@k8s-6-96 jenkins]# vi Dockerfile
FROM harbor.od.com/public/jenkins:v2.190.3
USER root
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
    echo 'Asia/Shanghai' >/etc/timezone
ADD id_rsa /root/.ssh/id_rsa
ADD config.json /root/.docker/config.json
ADD get-docker.sh /get-docker.sh
RUN echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&\
    /get-docker.sh --mirror=Aliyun

　　这个Dockerfile里我们主要做了以下几件事：

设置容器用户为root
设置容器内的时区
将创建的ssh私钥加入（使用git拉代码是要用，配对的公钥配置在gitlab中）
加入了登陆自建harbor仓库的config文件
修改了ssh客户端的配置，不做指纹验证
安装一个docker的客户端

　　2.5：制作自定义镜像

//准备所需文件，拷贝至/data/dockerfile/jenkins
[root@k8s-6-96 ~]# cd /data/dockerfile/jenkins
[root@k8s-6-96 jenkins]# cp /root/.ssh/id_rsa .
[root@k8s-6-96 jenkins]# cp /root/.docker/config.json .
[root@k8s-6-96 jenkins]# cp /root/get-docker.sh .

// 查看目录下的文件
[root@k8s-6-96 jenkins]# ll
total 32
-rw------- 1 root root   151 Nov 30 18:35 config.json
-rw-r--r-- 1 root root   349 Nov 30 18:31 Dockerfile
-rwxr-xr-x 1 root root 13216 Nov 30 18:31 get-docker.sh
-rw------- 1 root root  1675 Nov 30 18:35 id_rsa

//执行build
[root@k8s-6-96 jenkins]# docker build . -t harbor.auth.com/infra/jenkins:v2.190.3

// 查看镜像
[root@k8s-6-96 jenkins]# docker images | grep jenkins

　　2.6：在harbor中创建infra仓库，并设置为私有

　　 2.7：在任意运算节点上创建kubernetes名称空间并在此创建secret

[root@k8s-6-94 ~]# kubectl create namespace infra
[root@k8s-6-94 ~]# kubectl create secret docker-registry harbor --docker-server=harbor.auth.com --docker-username=admin --docker-password=Harbor12345 -n infra

　　2.8：在运维主机上推送镜像

[root@k8s-6-96 jenkins]# docker  push harbor.auth.com/infra/jenkins:v2.190.3

3：准备共享存储，原因是：jenkins中数据需要进行持久化存储，使用nfs共享存储方式

　　3.1：需要在运维主机(192.168.6.96)上和运算节点(192.168.6.94,192.168.6.95)上安装nfs

　　　　注意：运算节点只需要安装nfs引擎，不需要做任何配置

// 安装nfs-utils
[root@k8s-6-96 ~]# yum install nfs-utils -y

// 配置nfs服务
[root@k8s-6-96 ~]# vi /etc/exports
/data/nfs-volume 192.168.6.0/24(rw,no_root_squash)

// 启动NFS服务(运算节点不需要启动nfs)
[root@k8s-6-96 ~]# mkdir -p /data/nfs-volume
[root@k8s-6-96 ~]# systemctl start nfs
[root@k8s-6-96 ~]# systemctl enable nfs

4：准备资源配置清单　

　　4.1：在运维主机(192.168.6.96)上准备资源配置清单

[root@k8s-6-96 ~]# cd /data/k8s-yaml/
[root@k8s-6-96 k8s-yaml]# mkdir /data/k8s-yaml/jenkins && mkdir /data/nfs-volume/jenkins_home && cd  jenkins

　　4.2：编写资源配置清单

[root@k8s-6-96 jenkins]# vi dp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: jenkins
  namespace: infra
  labels: 
    name: jenkins
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: jenkins
  template:
    metadata:
      labels: 
        app: jenkins 
        name: jenkins
    spec:
      volumes:
      - name: data
        nfs: 
          server: k8s-6-96
          path: /data/nfs-volume/jenkins_home
      - name: docker
        hostPath: 
          path: /run/docker.sock
          type: ''
      containers:
      - name: jenkins
        image: harbor.auth.com/infra/jenkins:v2.190.3
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          protocol: TCP
        env:
        - name: JAVA_OPTS
          value: -Xmx512m -Xms512m
        volumeMounts:
        - name: data
          mountPath: /var/jenkins_home
        - name: docker
          mountPath: /run/docker.sock
      imagePullSecrets:
      - name: harbor
      securityContext: 
        runAsUser: 0
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

注释：如果是私有仓库，一定要添加 imagePullSecrets，否则拉取不到镜像。

[root@k8s-6-96 jenkins]# vi svc.yaml
kind: Service
apiVersion: v1
metadata: 
  name: jenkins
  namespace: infra
spec:
  ports:
  - protocol: TCP
    port: 80
    targetPort: 8080
  selector:
    app: jenkins

[root@k8s-6-96 jenkins]# vi ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata: 
  name: jenkins
  namespace: infra
spec:
  rules:
  - host: jenkins.auth.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: jenkins
          servicePort: 80

　　4.3：应用资源配置清单

[root@k8s-6-94 ~]# kubectl apply -f http://k8s-yaml.auth.com/jenkins/dp.yaml
[root@k8s-6-94 ~]# kubectl apply -f http://k8s-yaml.auth.com/jenkins/svc.yaml
[root@k8s-6-94 ~]# kubectl apply -f http://k8s-yaml.auth.com/jenkins/ingress.yaml

[root@k8s-6-94 ~]# kubectl get pods -n infra
NAME                       READY   STATUS    RESTARTS   AGE
jenkins-54b8469cf9-v46cc   1/1     Running   0          168m

[root@k8s-6-94 ~]# kubectl get all  -n infra
NAME                           READY   STATUS    RESTARTS   AGE
pod/jenkins-54b8469cf9-v46cc   1/1     Running   0          169m

NAME              TYPE        CLUSTER-IP        EXTERNAL-IP   PORT(S)   AGE
service/jenkins   ClusterIP   10。100.183.210   <none>        80/TCP    2d21h

NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/jenkins   1/1     1            1           2d21h

NAME                                 DESIRED   CURRENT   READY   AGE
replicaset.apps/jenkins-54b8469cf9   1         1         1       2d18h

5：在DNS服务器(192.168.6.92)上添加域名

[root@k8s-6-92 ~]# vi /var/named/auth.com.zone
// serial 序列号进行+ 1

// 添加jenkins A记录
jenkins            A    192.168.6.89

[root@k8s-6-92 ~]# systemctl restart named
[root@k8s-6-92 ~]# dig -t A jenkins.auth.com @192.168.6.92 +short
192.168.6.89

二：配置Jenkins

　　1：浏览器访问：jenkins.auth.com，需要出去初始化密码

初始密码查看

[root@k8s-6-96 ~]# cat /data/nfs-volume/jenkins_home/secrets/initialAdminPassword

　　2：安装插件

　　3：设置用户，用户名：admin 密码：admin123

4：完成安装

　　5：使用admin登录

　　6：设置jenkins

　　　　6.1：设置configure global security，允许匿名用户访问

　　　　6.2：阻止跨域请求，勾去掉

　　6.3：安装流水线插件Blue-Ocean

　　　　在运维主机(192.168.6.96)上设置清华大学源，可以加速安装插件

cd /data/nfs-volume/jenkins_home/updates
sed -i 's/http:\/\/updates.jenkins-ci.org\/download/https:\/\/mirrors.tuna.tsinghua.edu.cn\/jenkins/g' default.json
sed -i 's/http:\/\/www.google.com/https:\/\/www.baidu.com/g' default.json

　　　　安装插件Blue-Ocean

7：最后的准备工作

　　7.1：检查jenkins容器里的docker客户端，验证当前用户，时区

　　7.2：验证kubernetes名称空间创建的secret是否可登陆到harbor仓库

三：部署maven软件

　　编译java，早些年用javac-->ant -->maven-->Gragle 在运维主机hdss7-200上二进制部署，这里部署maven-3.6.2版本 mvn命令是一个脚本

　　1：下载安装包，并进行安装

　　maven官方下载地址：https://archive.apache.org/dist/maven/

　　目录8u232是根据docker容器里的jenkins的jdk版本命名

[root@k8s-6-96 ~]# tar zxf apache-maven-3.6.1-bin.tar.gz
[root@k8s-6-96 ~]# mv apache-maven-3.6.1 /data/nfs-volume/jenkins_home/maven-3.6.1-8u232

[root@k8s-6-96 jenkins_home]# cd maven-3.6.1-8u232/
[root@k8s-6-96 maven-3.6.1-8u232]# ll
drwxr-xr-x 2 root root     97 5月  26 15:42 bin
drwxr-xr-x 2 root root     42 5月  26 15:42 boot
drwxr-xr-x 3  501 games    63 5月  26 15:46 conf
drwxr-xr-x 4  501 games  4096 5月  26 15:42 lib
-rw-r--r-- 1  501 games 13437 4月   5 2019 LICENSE
-rw-r--r-- 1  501 games   182 4月   5 2019 NOTICE
-rw-r--r-- 1  501 games  2533 4月   5 2019 README.txt

　　2：设置settings.xml国内镜像源

[root@k8s-6-96 ~]# vi /data/nfs-volume/jenkins_home/maven-3.6.1-8u232/conf/settings.xml
<mirror>
  <id>nexus-aliyun</id>
  <mirrorOf>*</mirrorOf>
  <name>Nexus aliyun</name>
  <url>http://maven.aliyun.com/nexus/content/groups/public</url>
</mirror>

　　3：制作jre底包镜像

1：从公网pull jre底包
[root@k8s-6-96 ~]# docker pull docker.io/stanleyws/jre8:8u112
[root@k8s-6-96 ~]# docker images |grep jre
stanleyws/jre8                     8u112                      fa3a085d6ef1        2 years ago         363MB
[root@k8s-6-96 ~]# docker tag fa3a085d6ef1 harbor.auth.com/public/jre:8u112

2：准备java-agent的jar包
    // java agent 收集jvm的信息,用于普罗米修斯监控
[root@k8s-6-96 ~]# wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar -O jmx_javaagent-0.3.1.jar

3：准备config.yml和entrypoint.sh
    //config.yml 普罗米修斯的监控匹配规则
    // docker运行的默认启动脚本entrypoint.sh
[root@k8s-6-96 ~]# vi config.yml
---
rules:
  - pattern: '.*'

[root@k8s-6-96 ~]# vi entrypoint.sh
#!/bin/sh
M_OPTS="-Duser.timezone=Asia/Shanghai -javaagent:/opt/prom/jmx_javaagent-0.3.1.jar=$(hostname -i):${M_PORT:-"12346"}:/opt/prom/config.yml"
C_OPTS=${C_OPTS}
JAR_BALL=${JAR_BALL}
exec java -jar ${M_OPTS} ${C_OPTS} ${JAR_BALL}

[root@k8s-6-96 ~]# chmod +x entrypoint.sh

注：C_OPTS:连接配置中心(Apollo)环境，以及Apollo的地址
   JAR_BALL:jar包的名称 

4：在harbor中创建base项目，并设置为公开
[root@k8s-6-96 ~]# docker push  harbor.auth.com/public/jre:8u112

5：创建Dockerfile
[root@k8s-6-96 ~]# cd /data/dockerfile/jre8/
[root@k8s-6-96 jre8]# mv /root/config.yml .
[root@k8s-6-96 jre8]# mv /root/entrypoint.sh .
[root@k8s-6-96 jre8]# mv /root/jmx_javaagent-0.3.1.jar .
[root@k8s-6-96 jre8]# vi Dockerfile 
FROM harbor.auth.com/public/jre8:8u112
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
    echo 'Asia/Shanghai' >/etc/timezone
ADD config.yml /opt/prom/config.yml
ADD jmx_javaagent-0.3.1.jar /opt/prom/
WORKDIR /opt/project_dir
ADD entrypoint.sh /entrypoint.sh
CMD ["/entrypoint.sh"]

6：构建镜像并上传到harbor仓库
[root@k8s-6-96 jre8]# docker build . -t harbor.auth.com/base/jre8:8u112
[root@k8s-6-96 jre8]# docker push  harbor.auth.com/base/jre8:8u112

四：jenkins构建流水线，从git上拉取代码，进行编译成jar包，最后上传到harbor镜像仓库中

　　4.1：git版本

　　　　登陆jenkins----->选择NEW-ITEM----->item name ：dubbo-demo----->Pipeline------>ok

需要保留多少此老的构建，这里设置，保留三天，30个

点击：“This project is parameterized”使用参数化构建jenkins

添加参数String Parameter：8个------Trim the string都勾选 app_name

流水线脚本：

pipeline {
  agent any 
    stages {
      stage('pull') { //get project code from repo 
        steps {
          sh "git clone ${params.git_repo} ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.app_name}/${env.BUILD_NUMBER} && git checkout ${params.git_ver}"
        }
      }
      stage('build') { //exec mvn cmd
        steps {
          sh "cd ${params.app_name}/${env.BUILD_NUMBER}  && /var/jenkins_home/maven-${params.maven}/bin/${params.mvn_cmd}"
        }
      }
      stage('package') { //move jar file into project_dir
        steps {
          sh "cd ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.target_dir} && mkdir project_dir && mv *.jar ./project_dir"
        }
      }
      stage('image') { //build image and push to registry
        steps {
          writeFile file: "${params.app_name}/${env.BUILD_NUMBER}/Dockerfile", text: """FROM harbor.auth.com/${params.base_image}
ADD ${params.target_dir}/project_dir /opt/project_dir"""
          sh "cd  ${params.app_name}/${env.BUILD_NUMBER} && docker build -t harbor.auth.com/${params.image_name}:${params.git_ver}_${params.add_tag} . && docker push harbor.auth.com/${params.image_name}:${params.git_ver}_${params.add_tag}"
        }
      }
    }
}

依次填入/选择：

app_name

dubbo-demo-service
image_name

app/dubbo-demo-service
git_repo

https://gitee.com/stanleywang/dubbo-demo-service.git
git_ver

master
add_tag

190117_1920
mvn_dir

/
target_dir

./dubbo-server/target
mvn_cmd

mvn clean package -Dmaven.test.skip=true
base_image

base/jre8:8u112
maven

3.6.0-8u181

点击Build进行构建，等待构建完成。

构建完成之后，在harbor中可以查看到上传的镜像

交付jar包项目到K8S中：

创建资源配置清单：

[root@k8s-9-96 ]# deployment.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: dubbo-demo-consumer
  namespace: app
  labels: 
    name: dubbo-demo-consumer
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: dubbo-demo-consumer
  template:
    metadata:
      labels: 
        app: dubbo-demo-consumer
        name: dubbo-demo-consumer
    spec:
      containers:
      - name: dubbo-demo-consumer
        image: harbor.od.com/app/dubbo-demo-consumer:master_190119_2015
        ports:
        - containerPort: 8080
          protocol: TCP
        - containerPort: 20880
          protocol: TCP
        env:
        - name: JAR_BALL
          value: dubbo-client.jar
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

[root@k8s-9-96 ]# cat svc.yaml
kind: Service
apiVersion: v1
metadata: 
  name: dubbo-demo-consumer
  namespace: app
spec:
  ports:
  - protocol: TCP
    port: 8080
    targetPort: 8080
  selector: 
    app: dubbo-demo-consumer
  clusterIP: None
  type: ClusterIP
  sessionAffinity: None

[root@k8s-9-96 ]# cat ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata: 
  name: dubbo-demo-consumer
  namespace: app
spec:
  rules:
  - host: demo.auth.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: dubbo-demo-consumer
          servicePort: 8080

在任意一台运算节点上应用资源配置清单：

[root@k8s-6-96 ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-consumer/deployment.yaml
deployment.extensions/dubbo-demo-consumer created
[root@k8s-6-96 ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-consumer/svc.yaml
service/dubbo-demo-consumer created
[root@k8s-6-96 ~]# kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-consumer/ingress.yaml
ingress.extensions/dubbo-demo-consumer created

　　4.2：SVN版本

　　　　如果连接的是SVN，构建方式几乎一样，同样是选择参数话构建

与git相比，去掉一个 git_ver 参数，修改流水线脚本

pipeline {
  agent any 
    stages {
      stage('pull') { //get project code from repo 
        steps {
            sh "mkdir ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.app_name}/${env.BUILD_NUMBER} && svn checkout ${params.svn_repo} . --username=liyuanping --password=liypxxxx"
        }
      }
      stage('build') { //exec mvn cmd
        steps {
          sh "cd ${params.app_name}/${env.BUILD_NUMBER}  && /var/jenkins_home/maven-${params.maven}/bin/${params.mvn_cmd}"
        }
      }
      stage('package') { //move jar file into project_dir
        steps {
          sh "cd ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.target_dir} && mkdir project_dir && mv *.jar ./project_dir"
        }
      }
      stage('image') { //build image and push to registry
        steps {
          writeFile file: "${params.app_name}/${env.BUILD_NUMBER}/Dockerfile", text: """FROM harbor.auth.com/${params.base_image}
ADD ${params.target_dir}/project_dir /opt/project_dir"""
          sh "cd  ${params.app_name}/${env.BUILD_NUMBER} && docker build -t harbor.auth.com/${params.image_name}:${params.add_tag} . && docker push harbor.auth.com/${params.image_name}:${params.add_tag}"
        }
      }
    }
}

到此，jenkins 部署已完成

posted @ 2020-05-27 18:08 为生活而努力阅读(1253) 评论(0) 收藏举报

刷新页面返回顶部

为生活而努力

学习K8S之路.6--- 在K8S中部署Jenkins，并使用Jenkins打包jar包

5：使用admin登录

公告

　　5：使用admin登录