Kubernetes Calico - IPIP & Crosss-Subnet & VxLan【2024-02-28 测试成功】
IPIP
# ipipMode: Always
[root@ca-k8s-master01 ~]# calicoctl node status
Calico process is running.
IPv4 BGP status
+----------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+----------------+-------------------+-------+----------+-------------+
| 192.168.40.121 | node-to-node mesh | up | 04:20:26 | Established |
| 192.168.40.122 | node-to-node mesh | up | 04:20:19 | Established |
+----------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
#
[root@ca-k8s-master01 ~]# kubectl get ippools -o yaml
apiVersion: v1
items:
- apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
creationTimestamp: "2024-01-14T16:03:32Z"
generation: 1
name: default-ipv4-ippool
resourceVersion: "937"
uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
blockSize: 26
cidr: 10.244.0.0/16
ipipMode: Always
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
kind: List
metadata:
resourceVersion: ""
[root@ca-k8s-master01 ~]# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=6
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-55c5f88dcb-dj4q8 1/1 Running 0 100s 10.244.36.3 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-dpqkg 1/1 Running 0 100s 10.244.132.195 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-gwnvs 1/1 Running 0 100s 10.244.132.196 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-sxlxw 1/1 Running 0 100s 10.244.132.197 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-tmtmj 1/1 Running 0 100s 10.244.36.2 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-z4ktc 1/1 Running 0 100s 10.244.36.4 ca-k8s-node01 <none> <none>
#master01
#通往node1 、 node2节点应用 通过 tunl0 隧道转发
[root@ca-k8s-master01 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
10.244.36.0 192.168.40.121 255.255.255.192 UG 0 0 0 tunl0
10.244.132.192 192.168.40.122 255.255.255.192 UG 0 0 0 tunl0
10.244.237.0 0.0.0.0 255.255.255.192 U 0 0 0 *
10.244.237.3 0.0.0.0 255.255.255.255 UH 0 0 0 cali5d40fb51058
10.244.237.4 0.0.0.0 255.255.255.255 UH 0 0 0 calidadbb6c5f32
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
[root@ca-k8s-master01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:0c:29:4f:d5:05 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
link/ether 02:42:43:2f:97:4b brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether ea:e2:f2:60:19:f4 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
link/ether 6a:b8:60:11:85:3e brd ff:ff:ff:ff:ff:ff
6: cali5d40fb51058@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: calidadbb6c5f32@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
8: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
#node1
[root@ca-k8s-node01 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
10.244.36.0 0.0.0.0 255.255.255.192 U 0 0 0 *
10.244.36.2 0.0.0.0 255.255.255.255 UH 0 0 0 califc74e2139d8
10.244.36.3 0.0.0.0 255.255.255.255 UH 0 0 0 cali55aeef544d0
10.244.36.4 0.0.0.0 255.255.255.255 UH 0 0 0 cali835b664a77d
10.244.132.192 192.168.40.122 255.255.255.192 UG 0 0 0 tunl0
10.244.237.0 192.168.40.120 255.255.255.192 UG 0 0 0 tunl0
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
[root@ca-k8s-node01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:0c:29:2d:0f:c2 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
link/ether 02:42:7d:7f:f3:2a brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 42:a7:39:13:43:9e brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
link/ether 52:a7:b2:20:05:05 brd ff:ff:ff:ff:ff:ff
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
7: califc74e2139d8@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
8: cali55aeef544d0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali835b664a77d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
#node2
[root@ca-k8s-node02 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
10.244.36.0 192.168.40.121 255.255.255.192 UG 0 0 0 tunl0
10.244.132.192 0.0.0.0 255.255.255.192 U 0 0 0 *
10.244.132.194 0.0.0.0 255.255.255.255 UH 0 0 0 cali3ccbed006e5
10.244.132.195 0.0.0.0 255.255.255.255 UH 0 0 0 califf5bced79c2
10.244.132.196 0.0.0.0 255.255.255.255 UH 0 0 0 cali384c265ecc6
10.244.132.197 0.0.0.0 255.255.255.255 UH 0 0 0 cali5a11320bc84
10.244.237.0 192.168.40.120 255.255.255.192 UG 0 0 0 tunl0
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
[root@ca-k8s-node02 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:0c:29:d3:d6:f7 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
link/ether 02:42:3f:a7:5d:9c brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 1a:98:cb:08:5f:28 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
link/ether 86:45:c4:a0:18:64 brd ff:ff:ff:ff:ff:ff
6: cali3ccbed006e5@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
8: califf5bced79c2@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali384c265ecc6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3测试虚拟网卡-对应关系
#node2
[root@ca-k8s-node02 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:0c:29:d3:d6:f7 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
link/ether 02:42:3f:a7:5d:9c brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 1a:98:cb:08:5f:28 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
link/ether 86:45:c4:a0:18:64 brd ff:ff:ff:ff:ff:ff
6: cali3ccbed006e5@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
8: califf5bced79c2@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali384c265ecc6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3
#举例：
#10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
# link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3
# link-netnsid 3 表示命名空间为3的
[root@ca-k8s-node02 ~]# ip netns list
cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 (id: 3)
cni-1fef2d7b-1723-5a0b-e3b2-263ad964f1be (id: 2)
cni-3942a5f2-d75d-ca5a-d9d8-f0b1a9ac2e19 (id: 1)
cni-c6d89404-5172-383e-6879-179f77150179 (id: 0)
#查询出 - 10.244.132.197 地址 正是 node2上节点应用地址
#[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-55c5f88dcb-dj4q8 1/1 Running 0 21m 10.244.36.3 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-dpqkg 1/1 Running 0 21m 10.244.132.195 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-gwnvs 1/1 Running 0 21m 10.244.132.196 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-sxlxw 1/1 Running 0 21m 10.244.132.197 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-tmtmj 1/1 Running 0 21m 10.244.36.2 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-z4ktc 1/1 Running 0 21m 10.244.36.4 ca-k8s-node01 <none> <none>
[root@ca-k8s-node02 ~]# ip netns exec cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1480
inet 10.244.132.197 netmask 255.255.255.255 broadcast 10.244.132.197
inet6 fe80::6090:31ff:fe95:52a prefixlen 64 scopeid 0x20<link>
ether 62:90:31:95:05:2a txqueuelen 0 (Ethernet)
RX packets 5 bytes 446 (446.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 8 bytes 656 (656.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1000 (Local Loopback)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
#可以看到 4: 代表 之前查到的数据对应 cali5a11320bc84@if4 => if4
#可以看到 if10 代表 之前查到的数据对应 10: cali5a11320bc84@if4: => 10:
#4: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
# link/ether 62:90:31:95:05:2a brd ff:ff:ff:ff:ff:ff link-netnsid 0
[root@ca-k8s-node02 ~]# ip netns exec cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether 62:90:31:95:05:2a brd ff:ff:ff:ff:ff:ff link-netnsid 0测试1.同一节点应用访问不通过隧道传输
#访问node2同一节点的应用测试不会通过隧道进行转发
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-55c5f88dcb-dj4q8 1/1 Running 0 31m 10.244.36.3 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-dpqkg 1/1 Running 0 31m 10.244.132.195 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-gwnvs 1/1 Running 0 31m 10.244.132.196 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-sxlxw 1/1 Running 0 31m 10.244.132.197 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-tmtmj 1/1 Running 0 31m 10.244.36.2 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-z4ktc 1/1 Running 0 31m 10.244.36.4 ca-k8s-node01 <none> <none>
#使用tcpdump 抓包
[root@ca-k8s-master01 ~]# kubectl exec -it demoapp-55c5f88dcb-sxlxw /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.132.196
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-gwnvs, ServerIP: 10.244.132.196!
[root@ca-k8s-node02 ~]# tcpdump -i cali384c265ecc6 -nn
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on cali384c265ecc6, link-type EN10MB (Ethernet), capture size 262144 bytes
12:57:16.822160 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [S], seq 1811880604, win 28800, options [mss 1440,sackOK,TS val 1986397 ecr 0,nop,wscale 7], length 0
12:57:16.822189 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [S.], seq 1933121412, ack 1811880605, win 28560, options [mss 1440,sackOK,TS val 1986397 ecr 1986397,nop,wscale 7], length 0
12:57:16.822201 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1986397 ecr 1986397], length 0
12:57:16.822240 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [P.], seq 1:79, ack 1, win 225, options [nop,nop,TS val 1986397 ecr 1986397], length 78: HTTP: GET / HTTP/1.1
12:57:16.822243 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [.], ack 79, win 224, options [nop,nop,TS val 1986397 ecr 1986397], length 0
12:57:16.823051 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 1:18, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986397], length 17: HTTP: HTTP/1.0 200 OK
12:57:16.823090 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823150 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 18:155, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 137: HTTP
12:57:16.823169 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 155, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823178 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 155:273, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 118: HTTP
12:57:16.823184 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 273, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823212 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [F.], seq 273, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823266 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [F.], seq 79, ack 274, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823272 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [.], ack 80, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 0测试2.不同节点通过隧道进行传输
#node2 应用 访问 node1上服务
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-55c5f88dcb-dj4q8 1/1 Running 0 31m 10.244.36.3 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-dpqkg 1/1 Running 0 31m 10.244.132.195 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-gwnvs 1/1 Running 0 31m 10.244.132.196 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-sxlxw 1/1 Running 0 31m 10.244.132.197 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-tmtmj 1/1 Running 0 31m 10.244.36.2 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-z4ktc 1/1 Running 0 31m 10.244.36.4 ca-k8s-node01 <none> <none>
#通过 node1 物理网卡 ens33 进行抓包 因为进行了封装
[root@ca-k8s-node01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:0c:29:2d:0f:c2 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
link/ether 02:42:7d:7f:f3:2a brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 42:a7:39:13:43:9e brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
link/ether 52:a7:b2:20:05:05 brd ff:ff:ff:ff:ff:ff
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/ipip 0.0.0.0 brd 0.0.0.0
7: califc74e2139d8@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
8: cali55aeef544d0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali835b664a77d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
#使用node2上应用 访问 node1上应用
[root@ca-k8s-master01 ~]# kubectl exec -it demoapp-55c5f88dcb-sxlxw /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.36.3
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-dj4q8, ServerIP: 10.244.36.3!
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.36.3
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-dj4q8, ServerIP: 10.244.36.3!
#node1上 查看从node2 来的报文 是通过隧道进行转发传输的
[root@ca-k8s-node01 ~]# tcpdump -i ens33 -nn host 192.168.40.122
13:08:35.237744 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [S], seq 3784827341, win 28800, options [mss 1440,sackOK,TS val 2664811 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
13:08:35.237846 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [S.], seq 3640285741, ack 3784827342, win 28560, options [mss 1440,sackOK,TS val 2657861 ecr 2664811,nop,wscale 7], length 0 (ipip-proto-4)
13:08:35.237994 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 2664812 ecr 2657861], length 0 (ipip-proto-4)
13:08:35.238064 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 2664812 ecr 2657861], length 75: HTTP: GET / HTTP/1.1 (ipip-proto-4)
13:08:35.238128 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [.], ack 76, win 224, options [nop,nop,TS val 2657861 ecr 2664812], length 0 (ipip-proto-4)
13:08:35.238880 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 2657862 ecr 2664812], length 17: HTTP: HTTP/1.0 200 OK (ipip-proto-4)
13:08:35.239003 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 2657862 ecr 2664812], length 252: HTTP (ipip-proto-4)
13:08:35.239055 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 2664813 ecr 2657862], length 0 (ipip-proto-4)
13:08:35.239192 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 2664814 ecr 2657862], length 0 (ipip-proto-4)
13:08:35.239225 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [.], ack 77, win 224, options [nop,nop,TS val 2657862 ecr 2664814], length 0 (ipip-proto-4)
13:08:35.239674 IP 192.168.40.122.10250 > 192.168.40.120.60770: Flags [P.], seq 152:302, ack 252, win 295, options [nop,nop,TS val 2664814 ecr 2631591], length 150
13:08:35.239690 IP 192.168.40.122.10250 > 192.168.40.120.60770: Flags [P.], seq 302:367, ack 252, win 295, options [nop,nop,TS val 2664814 ecr 2631591], length 65CrossSubnet
#CrossSubnet
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml > default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# vim default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# cat default-ipv4-ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
creationTimestamp: "2024-01-14T16:03:32Z"
generation: 1
name: default-ipv4-ippool
resourceVersion: "937"
uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
blockSize: 26
cidr: 10.244.0.0/16
ipipMode: CrossSubnet
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
[root@ca-k8s-master01 ~]# kubectl apply -f default-ipv4-ippool.yaml
Warning: resource ippools/default-ipv4-ippool is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
ippool.crd.projectcalico.org/default-ipv4-ippool configured
#确认是否修改成功
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{"projectcalico.org/metadata":"{\"uid\":\"d1166c1a-f345-4c41-beb0-86a71ee0f32d\",\"creationTimestamp\":\"2024-01-14T16:03:32Z\"}"},"creationTimestamp":"2024-01-14T16:03:32Z","generation":1,"name":"default-ipv4-ippool","resourceVersion":"937","uid":"8615a080-3692-460c-91a1-b3eb5cb8c531"},"spec":{"blockSize":26,"cidr":"10.244.0.0/16","ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()","vxlanMode":"Never"}}
projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
creationTimestamp: "2024-01-14T16:03:32Z"
generation: 2
name: default-ipv4-ippool
resourceVersion: "8074"
uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
blockSize: 26
cidr: 10.244.0.0/16
ipipMode: CrossSubnet
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
#
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
demoapp-55c5f88dcb-dj4q8 1/1 Running 0 74m 10.244.36.3 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-dpqkg 1/1 Running 0 74m 10.244.132.195 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-gwnvs 1/1 Running 0 74m 10.244.132.196 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-sxlxw 1/1 Running 0 74m 10.244.132.197 ca-k8s-node02 <none> <none>
demoapp-55c5f88dcb-tmtmj 1/1 Running 0 74m 10.244.36.2 ca-k8s-node01 <none> <none>
demoapp-55c5f88dcb-z4ktc 1/1 Running 0 74m 10.244.36.4 ca-k8s-node01 <none> <none>
#查看route表 隧道turnl0 已经不用了 使用了 路由直接转发通过 ens33接口
[root@ca-k8s-master01 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.40.2 0.0.0.0 UG 100 0 0 ens33
10.244.36.0 192.168.40.121 255.255.255.192 UG 0 0 0 ens33
10.244.132.192 192.168.40.122 255.255.255.192 UG 0 0 0 ens33
10.244.237.0 0.0.0.0 255.255.255.192 U 0 0 0 *
10.244.237.3 0.0.0.0 255.255.255.255 UH 0 0 0 cali5d40fb51058
10.244.237.4 0.0.0.0 255.255.255.255 UH 0 0 0 calidadbb6c5f32
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
192.168.40.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
#通过node2 服务 访问 node1中服务 通过子网查看你
[root@ca-k8s-node01 ~]# tcpdump -i ens33 -nn net 10.244.0.0/16
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens33, link-type EN10MB (Ethernet), capture size 262144 bytes
13:44:29.479621 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [S], seq 2023735837, win 28800, options [mss 1440,sackOK,TS val 4819055 ecr 0,nop,wscale 7], length 0
13:44:29.479716 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [S.], seq 3743826456, ack 2023735838, win 28560, options [mss 1440,sackOK,TS val 4812103 ecr 4819055,nop,wscale 7], length 0
13:44:29.479863 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 4819055 ecr 4812103], length 0
13:44:29.479879 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 4819055 ecr 4812103], length 75: HTTP: GET / HTTP/1.1
13:44:29.479910 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [.], ack 76, win 224, options [nop,nop,TS val 4812103 ecr 4819055], length 0
13:44:29.480841 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 4812104 ecr 4819055], length 17: HTTP: HTTP/1.0 200 OK
13:44:29.480966 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 4812104 ecr 4819055], length 252: HTTP
13:44:29.481088 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 4819057 ecr 4812104], length 0
13:44:29.481178 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 4819057 ecr 4812104], length 0
13:44:29.481245 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [.], ack 77, win 224, options [nop,nop,TS val 4812105 ecr 4819057], length 0
13:44:30.307500 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [S], seq 3583458513, win 28800, options [mss 1440,sackOK,TS val 4819883 ecr 0,nop,wscale 7], length 0
13:44:30.307583 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [S.], seq 2010759158, ack 3583458514, win 28560, options [mss 1440,sackOK,TS val 4812931 ecr 4819883,nop,wscale 7], length 0
13:44:30.307749 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 4819883 ecr 4812931], length 0
13:44:30.307792 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 4819883 ecr 4812931], length 75: HTTP: GET / HTTP/1.1
13:44:30.307806 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [.], ack 76, win 224, options [nop,nop,TS val 4812931 ecr 4819883], length 0
13:44:30.308570 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 4812932 ecr 4819883], length 17: HTTP: HTTP/1.0 200 OK
13:44:30.308699 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 4812932 ecr 4819883], length 252: HTTP
13:44:30.308754 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 4819884 ecr 4812932], length 0
13:44:30.308848 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 4819884 ecr 4812932], length 0
13:44:30.308884 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [.], ack 77, win 224, options [nop,nop,TS val 4812932 ecr 4819884], length 0VxLan-此实验可能由于版本原因未成功 故截图
#改为： vxlanMode: Always
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml > default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# vim default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# cat default-ipv4-ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{"projectcalico.org/metadata":"{\"uid\":\"d1166c1a-f345-4c41-beb0-86a71ee0f32d\",\"creationTimestamp\":\"2024-01-14T16:03:32Z\"}"},"creationTimestamp":"2024-01-14T16:03:32Z","generation":1,"name":"default-ipv4-ippool","resourceVersion":"937","uid":"8615a080-3692-460c-91a1-b3eb5cb8c531"},"spec":{"blockSize":26,"cidr":"10.244.0.0/16","ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()","vxlanMode":"Never"}}
projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
creationTimestamp: "2024-01-14T16:03:32Z"
generation: 2
name: default-ipv4-ippool
resourceVersion: "8074"
uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
blockSize: 26
cidr: 10.244.0.0/16
ipipMode: Never
natOutgoing: true
nodeSelector: all()
vxlanMode: Always
[root@ca-k8s-master01 ~]# kubectl apply -f default-ipv4-ippool.yaml
ippool.crd.projectcalico.org/default-ipv4-ippool configured
# 
Calico BGP RR配置示例
#在100个节点规模以上的Calico集群环境中，为提升iBGP的效率，通常应该建立Router Reflector。
#https://github.com/BirkhoffXia/k8s-learning/tree/master/ProjectCalico
浙公网安备 33010602011771号