k8s集群中部署 jenkins master slave

前提：已有分布式存储方案，准备1个 存储类StorageClass

第一步：创建pvc 

vim jenkins-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkinshome-pvc
  namespace: devops
spec:
  storageClassName: storage-jenkins
  accessModes:
    - ReadWriteMany
  resources: 
    requests:
      storage: 200Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mavenrepository-pvc
  namespace: devops
spec:
  storageClassName: storage-jenkins
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 50Gi

　　

应用

kubectl apply -f jenkins-pvc.yaml

第二步：部署master。

（1）下载最新版的 jenkins.war 和 jdk 文件

（2）编写Dockerfile

vim Dockerfile

FROM centos:latest

ADD jdk-8u261-linux-x64.tar.gz /usr/java/jdk/
ADD jdk-7u80-linux-x64.tar.gz /usr/java/jdk/
ADD jenkins.war /usr/java/jenkins.war
ADD apache-maven /usr/local/tools/apache-maven

ENV LANG C.UTF-8
ENV TZ "Asia/Shanghai"
ENV JAVA_HOME /usr/java/jdk/jdk1.8.0_261
ENV PATH $PATH:$JAVA_HOME/bin
RUN yum -y install  unzip git subversion

ENTRYPOINT ["java","-Xms2048m", "-Xmx2048m","-Duser.home=/var/jenkins_home","-Dorg.apache.commons.jelly.tags.fmt.timeZone=Asia/Shanghai", "-Duser.timezone=Asia/Shanghai","-Djenkins.model.Jenkins.slaveAgentPort=50000","-jar","/usr/java/jenkins.war"]

 

 （3）构建镜像 

docker build -t yourImageName .

（4）推送镜像

push yourImageName

（5）编写rabc权限文件

vi jenkins-rabc.yaml

 

 

 

 

# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: yourname
  namespace: yournamespace

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: yourname
  namespace: yournamespace
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: yourname
  namespace: yournamespace
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: yourname
subjects:
- kind: ServiceAccount
  namespace: yournamespace
  name: yourname

　　kubectl apply -f  jenkins-rabc.yaml

（6）编写部署文件

vim jenkins-master.yaml

apiVersion: v1
kind: Service
metadata:
  name: yourname
  namespace: yournamespace
  labels:
    k8s.eip.work/layer: web
    k8s.eip.work/name: yourname
spec:
  selector:
    app: yourname
    release: canary
  #type: NodePort
  ports:
  - name: http
    targetPort: 8080
    port: 8080
    #nodePort: 30888
  - name: agent
    targetPort: 50000
    port: 50000
    #nodePort: 30500
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: yourname
  namespace: yournamespace
  labels:
    k8s.eip.work/layer: web
    k8s.eip.work/name: yourname
spec:
  replicas: 1
  selector: 
    matchLabels:
      app: yourname
      release: canary
  template:
    metadata:
      labels:
        app: yourname
        release: canary
    spec:
      #nodeName: k8s-node1
      serviceAccount: yourcount
      imagePullSecrets:
      - name: yoursecret
      containers:
      - name: yourname
        image: yourimage
        ports: 
        - name: http
          containerPort: 8080
          hostPort: 5088
          protocol: TCP
        - name: agent
          containerPort: 50000
          #hostPort: 50000
          protocol: TCP
        volumeMounts:
        - name: mavenrepository
          mountPath : /usr/local/tools/apache-maven/repository
        - name: jenkinshome
          mountPath : /var/jenkins_home
        - name: docker
          mountPath : /usr/bin/docker
        - name: docker-sock
          mountPath : /var/run/docker.sock
      volumes:
        - name: mavenrepository
          persistentVolumeClaim:
            claimName: mavenrepository-pvc
        - name: jenkinshome
          persistentVolumeClaim:
            claimName: jenkinshome-pvc
        - name: docker
          hostPath:
            path: /usr/bin/docker
        - name: docker-sock
          hostPath:
            path: /var/run/docker.sock

　　kubectl apply -f jenkins-master.yaml

第三步：构建slave镜像

（1）下载好slave.jar jdk jenkins-slave kubectl

（2）编写构建文件

vim Dockerfile

FROM centos:7
MAINTAINER diodonfrost <diodon.frost@diodonfrost.me>

ENV HOME=/home/jenkins
ARG VERSION=4.5

# Install systemd -- See https://hub.docker.com/_/centos/
RUN yum -y swap -- remove fakesystemd -- install systemd systemd-libs
#RUN yum -y update; \
#(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
#rm -f /lib/systemd/system/multi-user.target.wants/*; \
#rm -f /etc/systemd/system/*.wants/*; \
#rm -f /lib/systemd/system/local-fs.target.wants/*; \
#rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
#rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
#rm -f /lib/systemd/system/basic.target.wants/*; \
#rm -f /lib/systemd/system/anaconda.target.wants/*;

RUN yum -y install epel-release
RUN yum -y install unzip git subversion sudo 

RUN sed -i -e 's/^\(Defaults\s*requiretty\)/#--- \1/'  /etc/sudoers

ADD jdk-8u261-linux-x64.tar.gz /usr/java/jdk/
ADD jdk-7u80-linux-x64.tar.gz /usr/java/jdk/
ADD apache-maven /usr/local/tools/apache-maven
COPY kubectl /usr/bin/kubectl

ENV LANG C.UTF-8
ENV TZ "Asia/Shanghai"
ENV JAVA_HOME /usr/java/jdk/jdk1.8.0_261
ENV MAVEN_HOME /usr/local//usr/local/tools/apache-maven
ENV CLASSPATH .:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV PATH ${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${PATH}

# Add Jenkins user and group
RUN groupadd -g 10000 jenkins \
    && useradd -d $HOME -u 10000 -g jenkins jenkins

# Install jenkins jnlp
#RUN curl --create-dirs -sSLo /usr/share/jenkins/slave.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \
#    && chmod 755 /usr/share/jenkins \
#    && chmod 644 /usr/share/jenkins/slave.jar
ADD slave.jar /usr/share/jenkins/slave.jar
RUN  chmod 755 /usr/share/jenkins && chmod 644 /usr/share/jenkins/slave.jar
COPY jenkins-slave /usr/local/bin/jenkins-slave
RUN chmod 755 /usr/local/bin/jenkins-slave && chown jenkins:jenkins /usr/local/bin/jenkins-slave

RUN mkdir /home/jenkins/.jenkins \
    && mkdir -p /home/jenkins/agent \
    && chown -R jenkins:jenkins /home/jenkins

VOLUME /home/jenkins/.jenkins
VOLUME /home/jenkins/agent

# Systemd volume
VOLUME ["/sys/fs/cgroup"]

WORKDIR /home/jenkins

ENTRYPOINT ["/usr/local/bin/jenkins-slave"]

　　(3)构建镜像

docker build -t yourimagename .

第四步：赋予 slave rabc权限

vim slave.rabc.yaml

# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: yourname

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: yourname
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: yourname
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: yourname
subjects:
- kind: ServiceAccount
  name: yourname

　　kubectl apply -f slave.rabc.yaml

第五步：配置jenkins master

（1）安装kubernetes插件

（2）配置云

 

 

 

 

 

 

第六步：使用slave部署 

vim Jenkinsfile

pipeline {
    agent {label 'yourLabelName'}
    stages {
      stage("build"){

    }
}

　　

要注意的一些点：

1，创建master的rabc权限生成ServiceAccount账户，以生成的ServiceAccount账户身份启动 master容器，master账户就有了 对应的 kubectl执行权限。

2，每次增加新的namespace需要给 ServiceAccount 和 imagePullSecrets 绑定新的 namespace权限。

3，配置的slave的标签要记住 后面 Jenkinsfile会用到。

4，slave.jar 下载地址  http://yourserver:port/jnlpJars/slave.jar

 

参考文档

https://github.com/diodonfrost/docker-jenkins-slave