Service:
模型: userspace, iptables, ipvs
ClusterIP, NodePort
NodePort: client --> NodeIP:NodePort --> ClusterIP:ServicePort --> PodIP:containerPort
LoadBalancer
ExternelName
FQDN
CNAME --> FQDN
No ClusterIP: Headless Service
ServiceName --> PodIP
DaemonSet, Ingress Controller
HaProxy
Nginx
Traefik
Envoy
Ingress调度方式：
master:
[root@master manifests]# kubectl explain ingress [root@master manifests]# kubectl explain ingress.spec [root@master manifests]# kubectl explain ingress.spec.rules [root@master manifests]# kubectl explain ingress.spec.rules.http [root@master manifests]# kubectl explain ingress.spec.backend
部署Ingress Controller
官方网站：https://github.com/kubernetes/ingress-nginx
官网文档：https://kubernetes.github.io/ingress-nginx/deploy/#generic-deployment
下载NGINX: 0.17.1版本：https://github.com/kubernetes/ingress-nginx/releases?after=nginx-0.24.0
[root@master ~]# curl -o ingress-nginx-nginx-0.17.1.tar.gz https://codeload.github.com/kubernetes/ingress-nginx/tar.gz/nginx-0.17.1
[root@master ~]# ll
总用量 8652
-rw-------. 1 root root 1269 5月 4 2019 anaconda-ks.cfg
-rw-r--r--. 1 root root 171 4月 28 19:14 flannel.sh
-rw-r--r--. 1 root root 329 4月 27 20:23 image.sh
drwxr-xr-x. 2 root root 6 5月 22 20:44 ingress-nginx
-rw-r--r--. 1 root root 8824378 5月 22 20:39 ingress-nginx-nginx-0.17.1.tar.gz
-rw-r--r--. 1 root root 2312 4月 28 20:05 kube-flannel-legacy.yml
-rw-r--r--. 1 root root 14366 4月 27 22:01 kube-flannel.yml
drwxr-xr-x. 2 root root 278 5月 17 22:01 manifests
[root@master ~]# yum -y install git
[root@master ~]# kubectl create --help
[root@master ~]# kubectl create namespace dev
[root@master ~]# kubectl get ns
NAME STATUS AGE
default Active 25d
dev Active 13s
kube-public Active 25d
kube-system Active 25d
[root@master ~]# kubectl delete ns/dev
[root@master ~]# tar xf ingress-nginx-nginx-0.17.1.tar.gz
[root@master ~]# mkdir ingress-nginx
[root@master ~]# cd ingress-nginx-nginx-0.17.1/deploy/
[root@master deploy]# ll
总用量 40
-rw-rw-r--. 1 root root 134 7月 17 2018 configmap.yaml
-rw-rw-r--. 1 root root 1216 7月 17 2018 default-backend.yaml
drwxrwxr-x. 3 root root 24 7月 17 2018 grafana
-rw-rw-r--. 1 root root 6170 7月 17 2018 mandatory.yaml #可以通过这个yaml文件直接全部安装
-rw-rw-r--. 1 root root 68 7月 17 2018 namespace.yaml
drwxrwxr-x. 4 root root 60 7月 17 2018 provider
-rw-rw-r--. 1 root root 2390 7月 17 2018 rbac.yaml
-rw-rw-r--. 1 root root 80 7月 17 2018 README.md
-rw-rw-r--. 1 root root 94 7月 17 2018 tcp-services-configmap.yaml
-rw-rw-r--. 1 root root 94 7月 17 2018 udp-services-configmap.yaml
-rw-rw-r--. 1 root root 2174 7月 17 2018 with-rbac.yaml
[root@master deploy]# cp namespace.yaml configmap.yaml rbac.yaml tcp-services-configmap.yaml with-rbac.yaml udp-services-configmap.yaml default
-backend.yaml /root/ingress-nginx
[root@master deploy]# cd /root/ingress-nginx
[root@master ingress-nginx]# ll
总用量 28
-rw-r--r--. 1 root root 134 5月 28 20:14 configmap.yaml
-rw-r--r--. 1 root root 1216 5月 28 20:14 default-backend.yaml
-rw-r--r--. 1 root root 68 5月 28 20:14 namespace.yaml
-rw-r--r--. 1 root root 2390 5月 28 20:14 rbac.yaml
-rw-r--r--. 1 root root 94 5月 28 20:14 tcp-services-configmap.yaml
-rw-r--r--. 1 root root 94 5月 28 20:14 udp-services-configmap.yaml
-rw-r--r--. 1 root root 2174 5月 28 20:14 with-rbac.yaml
[root@master ingress-nginx]# kubectl apply -f namespace.yaml
[root@master ingress-nginx]# kubectl apply -f ./
[root@master deploy]# kubectl get pods -n ingress-nginx
NAME READY STATUS RESTARTS AGE
default-http-backend-846b65fb5f-ctbzr 0/1 ContainerCreating 0 21s
nginx-ingress-controller-d658896cd-ldbgc 1/1 Running 0 20s
[root@master ingress-nginx]# kubectl get pods -n ingress-nginx -w
NAME READY STATUS RESTARTS AGE
default-http-backend-846b65fb5f-nlrzm 0/1 ContainerCreating 0 11s
nginx-ingress-controller-d658896cd-xjzh2 0/1 Running 0 11s
nginx-ingress-controller-d658896cd-xjzh2 1/1 Running 0 11s
[root@master ~]# kubectl explain ingress.spec
[root@master ~]# cd manifests/
[root@master manifests]# mkdir ingress
[root@master manifests]# cd ingress/
[root@master ingress]# cp ../deploy-demo.yaml .
[root@master ingress]# vim deploy-demo.yaml
apiVersion: v1
kind: Service
metadata:
name: myapp
namespace: default
spec:
selector:
app: myapp
release: canary
ports:
- name: http
targetPort: 80
port: 80
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: myapp-deploy
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: myapp
release: canary
template:
metadata:
labels:
app: myapp
release: canary
spec:
containers:
- name: myapp
image: ikubernetes/myapp:v2
ports:
- name: http
containerPort: 80
[root@master ingress]# kubectl apply -f deploy-demo.yaml
[root@master ingress]# kubectl get pods
NAME READY STATUS RESTARTS AGE
myapp-deploy-67f6f6b4dc-ccb2j 1/1 Running 0 4s
myapp-deploy-67f6f6b4dc-d2kw8 1/1 Running 0 4s
myapp-deploy-67f6f6b4dc-gxvcx 1/1 Running 0 4s
[root@master ingress]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 31d
myapp ClusterIP 10.109.72.81 <none> 80/TCP 20s
无法下载镜像nginx-ingress-controller:0.17.0和defaultbackend:1.4导致创建Ingress Controller安装失败
defaultbackend:1.4镜像国内估计无法直接下载，要么FQ下载，要么在阿里云容器镜像服务https://cr.console.aliyun.com/us-west-1/instances/images搜索找到对应版本
node01:
[root@node01 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.17.0 [root@node01 ~]# docker pull gcr.io/google_containers/defaultbackend:1.4
node02:
[root@node02 ~]# docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.17.0 [root@node02 ~]# docker pull gcr.io/google_containers/defaultbackend:1.4
master:
[root@master ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
myapp-deploy-67f6f6b4dc-ccb2j 1/1 Running 0 17m
myapp-deploy-67f6f6b4dc-d2kw8 1/1 Running 0 17m
myapp-deploy-67f6f6b4dc-gxvcx 1/1 Running 0 17m
[root@master ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 31d
myapp ClusterIP 10.109.72.81 <none> 80/TCP 17m
[root@master ~]# cd ingress-nginx-nginx-0.17.1/deploy/provider/baremetal/
[root@master ingress-controller]# cp service-nodeport.yaml /root/ingress-nginx/
[root@master ingress-controller]# cd /root/ingress-nginx
[root@master ingress-nginx]# vim service-nodeport.yaml
apiVersion: v1
kind: Service
metadata:
name: ingress-nginx
namespace: ingress-nginx
spec:
type: NodePort
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
nodePort: 30080
- name: https
port: 443
targetPort: 443
protocol: TCP
nodePort: 30443
selector:
app: ingress-nginx
[root@master ingress-nginx]# kubectl apply -f service-nodeport.yaml
[root@master ingress-nginx]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default-http-backend ClusterIP 10.100.45.200 <none> 80/TCP 42m
ingress-nginx NodePort 10.98.247.197 <none> 80:30080/TCP,443:30443/TCP 21s
通过windws主机访问172.20.0.66:30080
master:
[root@master ingress-nginx]# cd ../manifests/ingress/
[root@master ~]# kubectl explain ingress
KIND: Ingress
VERSION: extensions/v1beta1
[root@master ~]# kubectl explain ingress.spec
[root@master ~]# kubectl explain ingress.spec.rules
[root@master ~]# kubectl explain ingress.spec.rules.http
[root@master ~]# kubectl explain ingress.spec.rules.http.paths
[root@master ~]# kubectl explain ingress.spec.rules.http.paths.backend
[root@master ~]# kubectl describe svc myapp
Name: myapp
Namespace: default
Labels: <none>
Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"name":
"myapp","namespace":"default"},"spec":{"ports":[{"name":"http","port":80,"targe...
Selector: app=myapp,release=canary
Type: ClusterIP
IP: 10.109.72.81
Port: http 80/TCP
TargetPort: 80/TCP
Endpoints: 10.244.1.56:80,10.244.1.57:80,10.244.2.47:80
Session Affinity: None
Events: <none>
[root@master ingress]# vim ingress-myapp.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-myapp
namespace: default
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: myapp.smoke.com
http:
paths:
- path:
backend:
serviceName: myapp
servicePort: 80
[root@master ingress]# kubectl apply -f ingress-myapp.yaml
[root@master ingress]# kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
ingress-myapp myapp.smoke.com 80 15s
[root@master ingress]# kubectl describe ingress ingress-myapp
Name: ingress-myapp
Namespace: default
Address:
Default backend: default-http-backend:80 (<none>)
Rules:
Host Path Backends
---- ---- --------
myapp.smoke.com
myapp:80 (<none>)
Annotations:
kubectl.kubernetes.io/last-applied-configuration: {"apiVersion":"extensions/v1beta1","kind":"Ingress","metadata":{"annotations":{"kubernetes
.io/ingress.class":"nginx"},"name":"ingress-myapp","namespace":"default"},"spec":{"rules":[{"host":"myapp.smoke.com","http":{"paths":[{"backend"
:{"serviceName":"myapp","servicePort":80},"path":null}]}}]}}
kubernetes.io/ingress.class: nginx
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal CREATE 45s nginx-ingress-controller Ingress default/ingress-myapp
[root@master ingress]# kubectl get pods -n ingress-nginx
NAME READY STATUS RESTARTS AGE
default-http-backend-846b65fb5f-nlrzm 1/1 Running 0 59m
nginx-ingress-controller-d658896cd-xjzh2 1/1 Running 0 59m
[root@master ingress]# kubectl exec -n ingress-nginx -it nginx-ingress-controller-d658896cd-xjzh2 -- /bin/sh
$ ls
fastcgi.conf fastcgi_params.default koi-win mime.types.default nginx.conf owasp-modsecurity-crs template
win-utf fastcgi.conf.default geoip lua modsecurity nginx.conf.default scgi_params
uwsgi_params fastcgi_params koi-utf mime.types modules opentracing.json scgi_params.default
uwsgi_params.default
$ cat nginx.conf
upstream default-myapp-80 {
least_conn;
keepalive 32;
server 10.244.2.47:80 max_fails=0 fail_timeout=0;
server 10.244.1.56:80 max_fails=0 fail_timeout=0;
server 10.244.1.57:80 max_fails=0 fail_timeout=0;
}
## start server myapp.smoke.com
server {
server_name myapp.smoke.com ;
listen 80;
listen [::]:80;
set $proxy_upstream_name "-";
location / {
set $namespace "default";
set $ingress_name "ingress-myapp";
set $service_name "myapp";
set $service_port "80";
set $location_path "/";
rewrite_by_lua_block {
}
log_by_lua_block {
monitor.call()
}
port_in_redirect off;
set $proxy_upstream_name "default-myapp-80";
client_max_body_size "1m";
proxy_set_header Host $best_http_host;
# Pass the extracted client certificate to the backend
# Allow websocket connections
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header X-Request-ID $req_id;
proxy_set_header X-Real-IP $the_real_ip;
proxy_set_header X-Forwarded-For $the_real_ip;
proxy_set_header X-Forwarded-Host $best_http_host;
proxy_set_header X-Forwarded-Port $pass_port;
proxy_set_header X-Forwarded-Proto $pass_access_scheme;
proxy_set_header X-Original-URI $request_uri;
proxy_set_header X-Scheme $pass_access_scheme;
# Pass the original X-Forwarded-For
proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;
# mitigate HTTPoxy Vulnerability
# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
proxy_set_header Proxy "";
# Custom headers to proxied server
proxy_connect_timeout 5s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
proxy_buffering "off";
proxy_buffer_size "4k";
proxy_buffers 4 "4k";
proxy_request_buffering "on";
proxy_http_version 1.1;
proxy_cookie_domain off;
proxy_cookie_path off;
# In case of errors try the next upstream server before returning an error
proxy_next_upstream error timeout;
proxy_next_upstream_tries 3;
proxy_pass http://default-myapp-80;
proxy_redirect off;
}
}
## end server myapp.smoke.com
通过windows主机访问myapp.smoke.com:30080，需要添加hosts文件C:\Windows\System32\drivers\etc\hosts；
172.20.0.66 myapp.smoke.com 172.20.0.67 myapp.smoke.com
测试结果
master:
[root@master ingress]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE
myapp-deploy-67f6f6b4dc-642hc 1/1 Running 0 1d 10.244.1.35 node01.smoke.com
myapp-deploy-67f6f6b4dc-d4cpv 1/1 Running 0 1d 10.244.2.33 node02.smoke.com
myapp-deploy-67f6f6b4dc-rqj8g 1/1 Running 0 1d 10.244.2.32 node02.smoke.com
[root@master ingress]# cp deploy-demo.yaml tomcat-deploy.yaml
[root@master ingress]# vim tomcat-deploy.yaml
apiVersion: v1
kind: Service
metadata:
name: tomcat
namespace: default
spec:
selector:
app: tomcat
release: canary
ports:
- name: http
targetPort: 8080
port: 8080
- name: ajp
targetPort: 8009
port: 8009
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tomcat-deploy
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: tomcat
release: canary
template:
metadata:
labels:
app: tomcat
release: canary
spec:
containers:
- name: tomcat
image: tomcat:8.5.32-jre8-alpine
ports:
- name: http
containerPort: 8080
- name: ajp
containerPort: 8009
[root@master ingress]# cp ingress-myapp.yaml ingress-tomcat.yaml
[root@master ingress]# vim ingress-tomcat.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-tomcat
namespace: default
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: tomcat.smoke.com
http:
paths:
- path:
backend:
serviceName: tomcat
servicePort: 8080
[root@master ingress]# kubectl apply -f tomcat-deploy.yaml
[root@master ingress]# kubectl exec tomcat-deploy-588c79d48d-6r8dw -- netstat -tnl
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State
tcp 0 0 0.0.0.0:8009 0.0.0.0:* LISTEN
tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN
tcp 0 0 127.0.0.1:8005 0.0.0.0:* LISTEN
[root@master ingress]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 6d
myapp ClusterIP 10.107.225.188 <none> 80/TCP 1d
tomcat ClusterIP 10.103.159.201 <none> 8080/TCP,8009/TCP 2m
[root@master ingress]# kubectl apply -f ingress-tomcat.yaml
[root@master ingress]# kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
ingress-myapp myapp.smoke.com 80 1d
ingress-tomcat tomcat.smoke.com 80 29s
[root@master ingress]# kubectl describe ingress ingress-tomcat
Name: ingress-tomcat
Namespace: default
Address:
Default backend: default-http-backend:80 (<none>)
Rules:
Host Path Backends
---- ---- --------
tomcat.smoke.com
tomcat:8080 (<none>)
Annotations:
kubectl.kubernetes.io/last-applied-configuration: {"apiVersion":"extensions/v1beta1","kind":"Ingress","metadata":{"annotations":{"kubernetes.io/ingress.class":"nginx"},"name":"ingress-tomcat","namespace":"defau
lt"},"spec":{"rules":[{"host":"tomcat.smoke.com","http":{"paths":[{"backend":{"serviceName":"tomcat","servicePort":8080},"path":null}]}}]}}
kubernetes.io/ingress.class: nginx
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal CREATE 1m nginx-ingress-controller Ingress default/ingress-tomcat
测试：编辑测试机/etc/hosts文件
smoke@smoke-GS70-2PC-Stealth:~$ sudo vim /etc/hosts 172.20.0.66 myapp.smoke.com tomcat.smoke.com 172.20.0.67 myapp.smoke.com tomcat.smoke.com
通过测试机器浏览器访问tomcat.smoke.com:30080
ingress调度https
master:
[root@master ingress]# openssl genrsa -out tls.key 2048
[root@master ingress]# ls
deploy-demo.yaml ingress-myapp.yaml ingress-tomcat.yaml tls.key tomcat-deploy.yaml
[root@master ingress]# openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=tomcat.smoke.com
[root@master ingress]# ls
deploy-demo.yaml ingress-myapp.yaml ingress-tomcat.yaml tls.crt tls.key tomcat-deploy.yaml
[root@master ingress]# kubectl create secret tls tomcat-ingress-secret --cert=tls.crt --key=tls.key
[root@master ingress]# kubectl get secret
NAME TYPE DATA AGE
default-token-jrx89 kubernetes.io/service-account-token 3 6d
tomcat-ingress-secret kubernetes.io/tls 2 20s
[root@master ingress]# kubectl describe secret tomcat-ingress-secret
Name: tomcat-ingress-secret
Namespace: default
Labels: <none>
Annotations: <none>
Type: kubernetes.io/tls
Data
====
tls.crt: 1294 bytes
tls.key: 1675 bytes
[root@master ingress]# kubectl explain ingress.spec
[root@master ingress]# kubectl explain ingress.spec.tls
[root@master ingress]# cp ingress-tomcat.yaml ingress-tomcat-tls.yaml
[root@master ingress]# vim ingress-tomcat-tls.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: ingress-tomcat-tls
namespace: default
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
tls:
- hosts:
- tomcat.smoke.com
secretName: tomcat-ingress-secret
rules:
- host: tomcat.smoke.com
http:
paths:
- path:
backend:
serviceName: tomcat
servicePort: 8080
[root@master ingress]# kubectl apply -f ingress-tomcat-tls.yaml
[root@master ingress]# kubectl get ingress
NAME HOSTS ADDRESS PORTS AGE
ingress-myapp myapp.smoke.com 80 1d
ingress-tomcat tomcat.smoke.com 80 19m
ingress-tomcat-tls tomcat.smoke.com 80, 443 19s
[root@master ingress]# kubectl describe ingress ingress-tomcat-tls
[root@master ingress]# kubectl exec -n ingress-nginx -it nginx-ingress-controller-d658896cd-krhh5 -- /bin/sh
$ cat nginx.conf
## start server tomcat.smoke.com
server {
server_name tomcat.smoke.com ;
listen 80;
listen [::]:80;
set $proxy_upstream_name "-";
listen 443 ssl http2;
listen [::]:443 ssl http2;
# PEM sha: 9be15816c3c3fdb48acf0fe61aa806fb5258a522
ssl_certificate /etc/ingress-controller/ssl/default-tomcat-ingress-secret.pem;
ssl_certificate_key /etc/ingress-controller/ssl/default-tomcat-ingress-secret.pem;
ssl_trusted_certificate /etc/ingress-controller/ssl/default-tomcat-ingress-secret-full-chain.pem;
ssl_stapling on;
ssl_stapling_verify on;
location / {
set $namespace "default";
set $ingress_name "ingress-tomcat";
set $service_name "tomcat";
set $service_port "8080";
set $location_path "/";
rewrite_by_lua_block {
}
log_by_lua_block {
monitor.call()
}
if ($scheme = https) {
more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains";
}
port_in_redirect off;
set $proxy_upstream_name "default-tomcat-8080";
# enforce ssl on server side
if ($redirect_to_https) {
return 308 https://$best_http_host$request_uri;
}
client_max_body_size "1m";
proxy_set_header Host $best_http_host;
# Pass the extracted client certificate to the backend
# Allow websocket connections
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header X-Request-ID $req_id;
proxy_set_header X-Real-IP $the_real_ip;
proxy_set_header X-Forwarded-For $the_real_ip;
proxy_set_header X-Forwarded-Host $best_http_host;
proxy_set_header X-Forwarded-Port $pass_port;
proxy_set_header X-Forwarded-Proto $pass_access_scheme;
proxy_set_header X-Original-URI $request_uri;
proxy_set_header X-Scheme $pass_access_scheme;
# Pass the original X-Forwarded-For
proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;
# mitigate HTTPoxy Vulnerability
# https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
proxy_set_header Proxy "";
# Custom headers to proxied server
proxy_connect_timeout 5s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
proxy_buffering "off";
proxy_buffer_size "4k";
proxy_buffers 4 "4k";
proxy_request_buffering "on";
proxy_http_version 1.1;
proxy_cookie_domain off;
proxy_cookie_path off;
# In case of errors try the next upstream server before returning an error
proxy_next_upstream error timeout;
proxy_next_upstream_tries 3;
proxy_pass http://default-tomcat-8080;
proxy_redirect off;
}
}
## end server tomcat.smoke.com
通过测试机器浏览器访问https://tomcat.smoke.com:30443
浙公网安备 33010602011771号