基于Ingress 实现集群服务访问

一、前言

Ingress与Ingress-controller

ingress对象：

指的是k8s中的一个api对象，一般用于yaml配置，如果集成了kubesphere，可以直接在UI界面上进行创建，其作用就是定义请求如何转发到service的规则，可以理解为配置模版

Ingress-controller

具体实现反向代理以及负载均衡的程序，对ingress定义的规则进行解析，根据配置的规则来实现请求转发，简单来说，ingress-controller才是负责具体转发的组建，通过各种方式将它暴露在集群入口，外部对集群的请求流量会先到ingress-controller，而ingress对象是用来告诉ingress-controller该如何转发请求，比如哪些哪些域名path路径要转发哪些服务等等

 

本章采用DaemonSet结合NodeSelector来部署Ingress-controller到特定的node节点上，然后使用HostNetwork直接 把该pod与宿主机node节点的网路打通，直接使用宿主机的80/443端口就能访问服务

此时，ingress-controller所在的node机器就很类似于传统架构的边缘节点，比如机房入口的nginx服务器，该方式整个请求链路都很简单，性能相对NodePort模式更好，缺点就是由于直接利用宿主机节点的网络和端口，一个node只能部署一个ingress-controller pod，比较适合大并发的生产环境使用

首先给指定的node节点打上标签，让我们的ingress-controller根据标签运行在指定的节点上，在这里我们选择k8s-node02节点

[root@k8s-master01 ingress]# kubectl label node k8s-node02 ingress=true

[root@k8s-master01 ingress]# kubectl get nodes --show-labels

 

 

部署nginx-ingress-controller

该mandatory.yaml文件中包含了很多资源的创建，例如namespace、Configmap、role、ServiceAccount等等所有部署ingress-controller需要的资源

#wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.25.0/deploy/static/mandatory.yaml 

对nginx-ingress-controller的yaml文件根据业务情况进行修改，在这改动的是ClusterRole资源配置、修改控制器类型为DaemonSet，并开启hostNetwork网络

#vim /opt/ingress/mandatory.yaml 

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"　　#nginx-ingress-controller:0.25版本增加network.k8s.io Ingress资源api
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"　　##nginx-ingress-controller:0.25版本增加network.k8s.io Ingress资源api
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: apps/v1
#kind: Deployment
kind: DaemonSet　　#将nginx-ingress-controller部署为Daemonset
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
#  replicas: 1　　#因为DaemonSet没有副本机制的概念，因此这里需要将其注释
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      hostNetwork: true　　#开启hostNetwork网络，让其使用宿主机节点的端口
      nodeSelector:　　　　#配置nodeSelector标签选择器
        ingress: "true"
      serviceAccountName: nginx-ingress-serviceaccount
      containers:
        - name: nginx-ingress-controller
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0
          args:
            - /nginx-ingress-controller
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10

---

 

[root@k8s-master01 ingress]# kubectl apply -f mandatory.yaml

[root@k8s-master01 ~]# kubectl get pod -n ingress-nginx -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP                NODE         NOMINATED NODE   READINESS GATES
nginx-ingress-controller-j4wq8   1/1     Running   0          20m   192.168.126.135   k8s-node02   <none>           <none>
[root@k8s-master01 ~]#  kubectl get cm,daemonset -n ingress-nginx -o wide
NAME                                        DATA   AGE
configmap/ingress-controller-leader-nginx   0      17m
configmap/kube-root-ca.crt                  1      20m
configmap/nginx-configuration               0      20m
configmap/tcp-services                      0      20m
configmap/udp-services                      0      20m

NAME                                      DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE   CONTAINERS                 IMAGES                                                                  SELECTOR
daemonset.apps/nginx-ingress-controller   1         1         1       1            1           ingress=true    20m   nginx-ingress-controller   quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0   app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/part-of=ingress-nginx

 在ingres-ingress-controller所在的node2节点上查看监听的端口以及容器服务

[root@k8s-node02 ~]# netstat -antp | grep nginx

 

 创建Ingress规则

这里我们启动了两个nginx pod作为后端的应用服务，以及指定暴露出service

[root@k8s-master01 ingress]# vim service-nginx.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-app-svc
spec:
  type: ClusterIP
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
  selector:
    app: nginx

#[root@k8s-master01 ingress]# kubectl get -f service-nginx.yaml

创建Ingress服务，

#vim ingressp-app.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-app-ingress
spec:
  rules:
  - host: www.test.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nginx-app-svc　　#指定svc名称
            port:
              number: 80

#kubectl apply -f  ingressp-app.yaml

 验证测试

在本地PC端配置一个host解析，测试访问

 

 假设我们需要给指定的域名加上https认证，那么该如何实现呢？

首先我们需要导入tsl证书，创建类型为tsl的secret, 将我们的证书和密钥导入进去

创建Secret

[root@k8s-master01 tsl]# kubectl create secret tls jiicm-com --key server.key --cert  jiicm.com.crt

查看详情

 配置ingres

#vim ingressp-app.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-app-ingress
spec:
  tls:
   - hosts:
       - ts.jiicm.com　　#指定https域名
     secretName: jiicm-com　　#指定类型为tls的secret名称（也就是上面我们创建的ssl证书）
  rules:
  - host: ts.jiicm.com
    http:
      paths:
      - path: /
        pathType: Prefix  #路径的匹配方式，目前有implementationSpecific、Exact和Prefix方式
        backend:
          service:
            name: nginx-app-svc　　#指定后端nginx服务的service名称
            port:
              number: 80

 

#kubectl replace -f  ingressp-app.yaml

 

 END！