#https://www.tigera.io/project-calico/
Calico:
Overlay:
IPIP: 原生支持的隧道协议
VXLAN：后来添加支持的隧道协议
Underlay：
三层路由：
路由表的生成：通过BGP协议学习生成
要求底层网络支持BGP报文
依赖于：部署应用程序，Daemon，从而将宿主机增强为支持BGP协议的路由器
Calico
后端：
隧道：
vxlan：完全不依赖于BGP
vxlan.calico
二层隧道：内部封装的是二层的帧；
ipip：依赖于BGP
tunl0
三层隧道：内部封装的三层IP报文；
路由：
bgp: iBGP
混合模式：
ipip, bgp
原生支持NetworkPolicy API
CRD:
增强版的NetworkPolicy
有一些，是提供Calico的自身维护和管理接口
地址池：
kubectl命令管理
Calico也额外提供了专用客户端：calicoctl
为Pod创建一个veth pair：
一端注入到Pod的网络名称空间中，作为网络接口使用；
另一端，保留在宿主机之上，直接作为专用于同该Pod通信的接口使用；而没有将其关联至一个专用的虚拟网桥上；
默认使用的网络：
192.168.0.0/16
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
取值：
Always：总是使用IPIP隧道协议封装跨节点的Pod间通信报文；
Cross-Subnet：
跨子的节点上的Pod间通信报文，使用IPIP隧道协议封装；
同一个二层网络中的节点上的Pod间的通信报文，直接使用路由模式；
路由表的生成：基于BGP协议进行；
Never：不使用IPIP隧道；
部署要点：
部署于kube-system名称空间中；
calico-node：
Felix
BIRD
calico-kube-controllers:
功能之一：NetworkPolicy的Controller;
#IPIP模式-通过隧道协议传输
#默认安装好calico会安装很多crd
[root@k8s-master01 ~]# kubectl get node k8s-master01 -o yaml
ippools.crd.projectcalico.org 2024-01-05T04:21:29Z
#查看默认配置calico ippools使用参数
#默认是分配子网为26位的 blockSize: 26
[root@k8s-master01 ~]# kubectl get ippools
NAME AGE
default-ipv4-ippool 44d
[root@k8s-master01 ~]# kubectl get ippools -o yaml
apiVersion: v1
items:
- apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
projectcalico.org/metadata: '{"uid":"0f4974f6-6a3f-40e7-af7a-f3005b301a0b","creationTimestamp":"2024-01-05T04:21:33Z"}'
creationTimestamp: "2024-01-05T04:21:33Z"
generation: 1
name: default-ipv4-ippool
resourceVersion: "6813"
uid: 6e35e972-911f-4ede-8056-46a7d50111fd
spec:
blockSize: 26
cidr: 172.16.0.0/16
ipipMode: Always
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
kind: List
metadata:
resourceVersion: ""
#IPIP模式
# ipipMode: Always
ipipMode: Always
#安装calicoctl
#https://docs.tigera.io/calico/3.26/operations/calicoctl/install
#curl -L https://github.com/projectcalico/calico/releases/download/v3.26.4/calicoctl-linux-amd64 -o calicoctl
[root@k8s-master01 ~]# mv calicoctl-linux-amd64 calicoctl
[root@k8s-master01 ~]# chmod a+x calicoctl
[root@k8s-master01 ~]# mv calicoctl /usr/bin/
[root@k8s-master01 ~]# chmod +x ./calicoctl
[root@k8s-master01 ~]# calicoctl node status
Calico process is running.
IPv4 BGP status
+----------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+----------------+-------------------+-------+----------+-------------+
| 192.168.40.112 | node-to-node mesh | up | 02:05:49 | Established |
+----------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
#实验
#创建3个pod
[root@k8s-master01 ~]# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=3
[root@k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-7c58cd6bb-75pf8 1/1 Running 0 72s 172.16.32.129 k8s-master01 <none> <none>
demoapp-7c58cd6bb-dlv2l 1/1 Running 0 72s 172.16.58.203 k8s-node02 <none> <none>
demoapp-7c58cd6bb-kqtrf 1/1 Running 0 72s 172.16.58.202 k8s-node02 <none> <none>
#查看node2路由表 会生成相应记录 tunl0 表示隧道接口
#172.16.32.128 192.168.40.101 255.255.255.192 UG 0 0 0 tunl0 表示到达172.16.32.128 需要网关192.168.40.101 通过tunl0隧道转发
[root@k8s-node02 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
172.16.32.128 192.168.40.101 255.255.255.192 UG 0 0 0 tunl0
172.16.58.192 0.0.0.0 255.255.255.192 U 0 0 0 *
172.16.58.199 0.0.0.0 255.255.255.255 UH 0 0 0 cali4373dce91d4
172.16.58.200 0.0.0.0 255.255.255.255 UH 0 0 0 cali3d006ffa41c
172.16.58.201 0.0.0.0 255.255.255.255 UH 0 0 0 cali190b13b1251
172.16.58.202 0.0.0.0 255.255.255.255 UH 0 0 0 cali69ecdd194dd
172.16.58.203 0.0.0.0 255.255.255.255 UH 0 0 0 calieaff9349a21
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
#查看报文
#master上访问 node2上服务
[root@k8s-master01 ~]# kubectl exec -it demoapp-7c58cd6bb-75pf8 /bin/sh
[root@demoapp-7c58cd6bb-75pf8 /]# while true ;do curl 172.16.58.203;done
#node2抓包[root@k8s-node02 ~]# tcpdump -i ens33 -nn host 192.168.40.112 | grep 172.16.58.203.80
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens33, link-type EN10MB (Ethernet), capture size 262144 bytes
21:34:43.726299 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36092 > 172.16.58.203.80: Flags [S], seq 4140346692, win 28800, options [mss 1440,sackOK,TS val 1467744 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
21:34:43.726467 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36092: Flags [S.], seq 12199371, ack 4140346693, win 28560, options [mss 1440,sackOK,TS val 1509355 ecr 1467744,nop,wscale 7], length 0 (ipip-proto-4)
21:34:43.726704 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36092 > 172.16.58.203.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1467744 ecr 1509355], length 0 (ipip-proto-4)
21:34:43.726743 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36092 > 172.16.58.203.80: Flags [P.], seq 1:78, ack 1, win 225, options [nop,nop,TS val 1467745 ecr 1509355], length 77: HTTP: GET / HTTP/1.1 (ipip-proto-4)
21:34:43.726804 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36092: Flags [.], ack 78, win 224, options [nop,nop,TS val 1509355 ecr 1467745], length 0 (ipip-proto-4)
21:34:43.727617 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36092: Flags [P.], seq 1:18, ack 78, win 224, options [nop,nop,TS val 1509356 ecr 1467745], length 17: HTTP: HTTP/1.0 200 OK (ipip-proto-4)
21:34:43.727752 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36092: Flags [FP.], seq 18:270, ack 78, win 224, options [nop,nop,TS val 1509356 ecr 1467745], length 252: HTTP (ipip-proto-4)
21:34:43.727850 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36092 > 172.16.58.203.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 1467746 ecr 1509356], length 0 (ipip-proto-4)
21:34:43.727903 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36092 > 172.16.58.203.80: Flags [F.], seq 78, ack 271, win 234, options [nop,nop,TS val 1467746 ecr 1509356], length 0 (ipip-proto-4)
21:34:43.727922 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36092: Flags [.], ack 79, win 224, options [nop,nop,TS val 1509356 ecr 1467746], length 0 (ipip-proto-4)
21:34:44.230982 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36098 > 172.16.58.203.80: Flags [S], seq 3334202967, win 28800, options [mss 1440,sackOK,TS val 1468249 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
21:34:44.231097 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36098: Flags [S.], seq 892723298, ack 3334202968, win 28560, options [mss 1440,sackOK,TS val 1509860 ecr 1468249,nop,wscale 7], length 0 (ipip-proto-4)
21:34:44.231326 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36098 > 172.16.58.203.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1468249 ecr 1509860], length 0 (ipip-proto-4)
21:34:44.231332 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36098 > 172.16.58.203.80: Flags [P.], seq 1:78, ack 1, win 225, options [nop,nop,TS val 1468249 ecr 1509860], length 77: HTTP: GET / HTTP/1.1 (ipip-proto-4)
21:34:44.231385 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36098: Flags [.], ack 78, win 224, options [nop,nop,TS val 1509860 ecr 1468249], length 0 (ipip-proto-4)
21:34:44.232227 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36098: Flags [P.], seq 1:18, ack 78, win 224, options [nop,nop,TS val 1509861 ecr 1468249], length 17: HTTP: HTTP/1.0 200 OK (ipip-proto-4)
21:34:44.232353 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36098: Flags [FP.], seq 18:270, ack 78, win 224, options [nop,nop,TS val 1509861 ecr 1468249], length 252: HTTP (ipip-proto-4)
21:34:44.232359 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36098 > 172.16.58.203.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 1468250 ecr 1509861], length 0 (ipip-proto-4)
21:34:44.232522 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36098 > 172.16.58.203.80: Flags [F.], seq 78, ack 271, win 234, options [nop,nop,TS val 1468251 ecr 1509861], length 0 (ipip-proto-4)
21:34:44.232559 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36098: Flags [.], ack 79, win 224, options [nop,nop,TS val 1509861 ecr 1468251], length 0 (ipip-proto-4)
21:34:44.735428 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36102 > 172.16.58.203.80: Flags [S], seq 3038711296, win 28800, options [mss 1440,sackOK,TS val 1468754 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
21:34:44.735533 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36102: Flags [S.], seq 1616882981, ack 3038711297, win 28560, options [mss 1440,sackOK,TS val 1510364 ecr 1468754,nop,wscale 7], length 0 (ipip-proto-4)
21:34:44.735671 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36102 > 172.16.58.203.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1468754 ecr 1510364], length 0 (ipip-proto-4)
21:34:44.735734 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36102 > 172.16.58.203.80: Flags [P.], seq 1:78, ack 1, win 225, options [nop,nop,TS val 1468754 ecr 1510364], length 77: HTTP: GET / HTTP/1.1 (ipip-proto-4)
21:34:44.735750 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36102: Flags [.], ack 78, win 224, options [nop,nop,TS val 1510364 ecr 1468754], length 0 (ipip-proto-4)
21:34:44.736745 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36102: Flags [P.], seq 1:18, ack 78, win 224, options [nop,nop,TS val 1510365 ecr 1468754], length 17: HTTP: HTTP/1.0 200 OK (ipip-proto-4)
21:34:44.736872 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36102: Flags [FP.], seq 18:270, ack 78, win 224, options [nop,nop,TS val 1510365 ecr 1468754], length 252: HTTP (ipip-proto-4)
21:34:44.736895 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36102 > 172.16.58.203.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 1468755 ecr 1510365], length 0 (ipip-proto-4)
21:34:44.737004 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36102 > 172.16.58.203.80: Flags [F.], seq 78, ack 271, win 234, options [nop,nop,TS val 1468755 ecr 1510365], length 0 (ipip-proto-4)
21:34:44.737050 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36102: Flags [.], ack 79, win 224, options [nop,nop,TS val 1510366 ecr 1468755], length 0 (ipip-proto-4)
21:34:45.241346 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36104 > 172.16.58.203.80: Flags [S], seq 78945047, win 28800, options [mss 1440,sackOK,TS val 1469259 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
21:34:45.241470 IP 192.168.40.112 > 192.168.40.101: IP 172.16.58.203.80 > 172.16.32.129.36104: Flags [S.], seq 2262026023, ack 78945048, win 28560, options [mss 1440,sackOK,TS val 1510870 ecr 1469259,nop,wscale 7], length 0 (ipip-proto-4)
21:34:45.241869 IP 192.168.40.101 > 192.168.40.112: IP 172.16.32.129.36104 > 172.16.58.203.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1469260 ecr 1510870], length 0 (ipip-proto-4)
#CrossSubnet模式-通过路由模式传输
#删除之前的pod
[root@k8s-master01 ~]# kubectl delete deployment demoapp
#获取ippools 修改 ipipMode: CrossSubnet
[root@k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml > default-ipv4-ippool.yaml
[root@k8s-master01 ~]# vim default-ipv4-ippool.yaml
[root@k8s-master01 ~]# cat default-ipv4-ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
projectcalico.org/metadata: '{"uid":"0f4974f6-6a3f-40e7-af7a-f3005b301a0b","creationTimestamp":"2024-01-05T04:21:33Z"}'
creationTimestamp: "2024-01-05T04:21:33Z"
generation: 1
name: default-ipv4-ippool
resourceVersion: "6813"
uid: 6e35e972-911f-4ede-8056-46a7d50111fd
spec:
blockSize: 26
cidr: 172.16.0.0/16
ipipMode: CrossSubnet
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
[root@k8s-master01 ~]# kubectl apply -f default-ipv4-ippool.yaml
#再次创建pod
[root@k8s-master01 ~]# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=3
deployment.apps/demoapp created
[root@k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-7c58cd6bb-5ttrh 1/1 Running 0 9s 172.16.58.205 k8s-node02 <none> <none>
demoapp-7c58cd6bb-qz2qc 1/1 Running 0 9s 172.16.32.130 k8s-master01 <none> <none>
demoapp-7c58cd6bb-wp2kt 1/1 Running 0 9s 172.16.58.204 k8s-node02 <none> <none>
#查看路由表 没有隧道了
[root@k8s-node02 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
172.16.32.128 192.168.40.101 255.255.255.192 UG 0 0 0 ens33
172.16.58.192 0.0.0.0 255.255.255.192 U 0 0 0 *
172.16.58.199 0.0.0.0 255.255.255.255 UH 0 0 0 cali4373dce91d4
172.16.58.200 0.0.0.0 255.255.255.255 UH 0 0 0 cali3d006ffa41c
172.16.58.201 0.0.0.0 255.255.255.255 UH 0 0 0 cali190b13b1251
172.16.58.204 0.0.0.0 255.255.255.255 UH 0 0 0 cali2d72f58ef6e
172.16.58.205 0.0.0.0 255.255.255.255 UH 0 0 0 cali0cc8fa4c25b
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
浙公网安备 33010602011771号