Kubernetes Calico - IPIP & Crosss-Subnet & VxLan【2024-02-28 测试成功】

IPIP

#    ipipMode: Always
[root@ca-k8s-master01 ~]# calicoctl node status
Calico process is running.

IPv4 BGP status
+----------------+-------------------+-------+----------+-------------+
|  PEER ADDRESS  |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+----------------+-------------------+-------+----------+-------------+
| 192.168.40.121 | node-to-node mesh | up    | 04:20:26 | Established |
| 192.168.40.122 | node-to-node mesh | up    | 04:20:19 | Established |
+----------------+-------------------+-------+----------+-------------+

IPv6 BGP status
No IPv6 peers found.

#
[root@ca-k8s-master01 ~]#  kubectl get ippools -o yaml
apiVersion: v1
items:
- apiVersion: crd.projectcalico.org/v1
  kind: IPPool
  metadata:
    annotations:
      projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
    creationTimestamp: "2024-01-14T16:03:32Z"
    generation: 1
    name: default-ipv4-ippool
    resourceVersion: "937"
    uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
  spec:
    blockSize: 26
    cidr: 10.244.0.0/16
    ipipMode: Always
    natOutgoing: true
    nodeSelector: all()
    vxlanMode: Never
kind: List
metadata:
  resourceVersion: ""


[root@ca-k8s-master01 ~]# kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=6
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME                       READY   STATUS    RESTARTS   AGE    IP               NODE            NOMINATED NODE   READINESS GATES
demoapp-55c5f88dcb-dj4q8   1/1     Running   0          100s   10.244.36.3      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-dpqkg   1/1     Running   0          100s   10.244.132.195   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-gwnvs   1/1     Running   0          100s   10.244.132.196   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-sxlxw   1/1     Running   0          100s   10.244.132.197   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-tmtmj   1/1     Running   0          100s   10.244.36.2      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-z4ktc   1/1     Running   0          100s   10.244.36.4      ca-k8s-node01   <none>           <none>

#master01
#通往node1 、 node2节点应用 通过 tunl0 隧道转发
[root@ca-k8s-master01 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.40.2    0.0.0.0         UG    100    0        0 ens33
10.244.36.0     192.168.40.121  255.255.255.192 UG    0      0        0 tunl0
10.244.132.192  192.168.40.122  255.255.255.192 UG    0      0        0 tunl0
10.244.237.0    0.0.0.0         255.255.255.192 U     0      0        0 *
10.244.237.3    0.0.0.0         255.255.255.255 UH    0      0        0 cali5d40fb51058
10.244.237.4    0.0.0.0         255.255.255.255 UH    0      0        0 calidadbb6c5f32
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.40.0    0.0.0.0         255.255.255.0   U     100    0        0 ens33

[root@ca-k8s-master01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:4f:d5:05 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:43:2f:97:4b brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether ea:e2:f2:60:19:f4 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
    link/ether 6a:b8:60:11:85:3e brd ff:ff:ff:ff:ff:ff
6: cali5d40fb51058@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: calidadbb6c5f32@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
8: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    
#node1
[root@ca-k8s-node01 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.40.2    0.0.0.0         UG    100    0        0 ens33
10.244.36.0     0.0.0.0         255.255.255.192 U     0      0        0 *
10.244.36.2     0.0.0.0         255.255.255.255 UH    0      0        0 califc74e2139d8
10.244.36.3     0.0.0.0         255.255.255.255 UH    0      0        0 cali55aeef544d0
10.244.36.4     0.0.0.0         255.255.255.255 UH    0      0        0 cali835b664a77d
10.244.132.192  192.168.40.122  255.255.255.192 UG    0      0        0 tunl0
10.244.237.0    192.168.40.120  255.255.255.192 UG    0      0        0 tunl0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.40.0    0.0.0.0         255.255.255.0   U     100    0        0 ens33

[root@ca-k8s-node01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:2d:0f:c2 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:7d:7f:f3:2a brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 42:a7:39:13:43:9e brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
    link/ether 52:a7:b2:20:05:05 brd ff:ff:ff:ff:ff:ff
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
7: califc74e2139d8@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
8: cali55aeef544d0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali835b664a77d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2

#node2
[root@ca-k8s-node02 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.40.2    0.0.0.0         UG    100    0        0 ens33
10.244.36.0     192.168.40.121  255.255.255.192 UG    0      0        0 tunl0
10.244.132.192  0.0.0.0         255.255.255.192 U     0      0        0 *
10.244.132.194  0.0.0.0         255.255.255.255 UH    0      0        0 cali3ccbed006e5
10.244.132.195  0.0.0.0         255.255.255.255 UH    0      0        0 califf5bced79c2
10.244.132.196  0.0.0.0         255.255.255.255 UH    0      0        0 cali384c265ecc6
10.244.132.197  0.0.0.0         255.255.255.255 UH    0      0        0 cali5a11320bc84
10.244.237.0    192.168.40.120  255.255.255.192 UG    0      0        0 tunl0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.40.0    0.0.0.0         255.255.255.0   U     100    0        0 ens33

[root@ca-k8s-node02 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:d3:d6:f7 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:3f:a7:5d:9c brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 1a:98:cb:08:5f:28 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
    link/ether 86:45:c4:a0:18:64 brd ff:ff:ff:ff:ff:ff
6: cali3ccbed006e5@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
8: califf5bced79c2@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali384c265ecc6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3

测试虚拟网卡-对应关系

#node2
[root@ca-k8s-node02 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:d3:d6:f7 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:3f:a7:5d:9c brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 1a:98:cb:08:5f:28 brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
    link/ether 86:45:c4:a0:18:64 brd ff:ff:ff:ff:ff:ff
6: cali3ccbed006e5@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
7: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
8: califf5bced79c2@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali384c265ecc6@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2
10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3

#举例:
#10: cali5a11320bc84@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
#    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 3
# link-netnsid 3 表示命名空间为3的
[root@ca-k8s-node02 ~]# ip netns list
cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 (id: 3)
cni-1fef2d7b-1723-5a0b-e3b2-263ad964f1be (id: 2)
cni-3942a5f2-d75d-ca5a-d9d8-f0b1a9ac2e19 (id: 1)
cni-c6d89404-5172-383e-6879-179f77150179 (id: 0)

#查询出 - 10.244.132.197 地址 正是 node2上节点应用地址
#[root@ca-k8s-master01 ~]# kubectl  get pods -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
demoapp-55c5f88dcb-dj4q8   1/1     Running   0          21m   10.244.36.3      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-dpqkg   1/1     Running   0          21m   10.244.132.195   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-gwnvs   1/1     Running   0          21m   10.244.132.196   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-sxlxw   1/1     Running   0          21m   10.244.132.197   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-tmtmj   1/1     Running   0          21m   10.244.36.2      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-z4ktc   1/1     Running   0          21m   10.244.36.4      ca-k8s-node01   <none>           <none>

[root@ca-k8s-node02 ~]# ip netns exec cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1480
        inet 10.244.132.197  netmask 255.255.255.255  broadcast 10.244.132.197
        inet6 fe80::6090:31ff:fe95:52a  prefixlen 64  scopeid 0x20<link>
        ether 62:90:31:95:05:2a  txqueuelen 0  (Ethernet)
        RX packets 5  bytes 446 (446.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 8  bytes 656 (656.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

#可以看到 4: 代表 之前查到的数据对应 cali5a11320bc84@if4 => if4
#可以看到 if10 代表 之前查到的数据对应 10: cali5a11320bc84@if4: => 10:
#4: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
#    link/ether 62:90:31:95:05:2a brd ff:ff:ff:ff:ff:ff link-netnsid 0
[root@ca-k8s-node02 ~]# ip netns exec cni-255cbfbf-af17-2810-8c22-2a20fa4a5341 ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether 62:90:31:95:05:2a brd ff:ff:ff:ff:ff:ff link-netnsid 0

测试1.同一节点应用访问不通过隧道传输

#访问node2同一节点的应用测试不会通过隧道进行转发
[root@ca-k8s-master01 ~]# kubectl  get pods -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
demoapp-55c5f88dcb-dj4q8   1/1     Running   0          31m   10.244.36.3      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-dpqkg   1/1     Running   0          31m   10.244.132.195   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-gwnvs   1/1     Running   0          31m   10.244.132.196   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-sxlxw   1/1     Running   0          31m   10.244.132.197   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-tmtmj   1/1     Running   0          31m   10.244.36.2      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-z4ktc   1/1     Running   0          31m   10.244.36.4      ca-k8s-node01   <none>           <none>
 
#使用tcpdump 抓包
[root@ca-k8s-master01 ~]# kubectl  exec -it demoapp-55c5f88dcb-sxlxw /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.132.196
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-gwnvs, ServerIP: 10.244.132.196!

[root@ca-k8s-node02 ~]# tcpdump -i cali384c265ecc6 -nn
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on cali384c265ecc6, link-type EN10MB (Ethernet), capture size 262144 bytes
12:57:16.822160 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [S], seq 1811880604, win 28800, options [mss 1440,sackOK,TS val 1986397 ecr 0,nop,wscale 7], length 0
12:57:16.822189 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [S.], seq 1933121412, ack 1811880605, win 28560, options [mss 1440,sackOK,TS val 1986397 ecr 1986397,nop,wscale 7], length 0
12:57:16.822201 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 1986397 ecr 1986397], length 0
12:57:16.822240 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [P.], seq 1:79, ack 1, win 225, options [nop,nop,TS val 1986397 ecr 1986397], length 78: HTTP: GET / HTTP/1.1
12:57:16.822243 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [.], ack 79, win 224, options [nop,nop,TS val 1986397 ecr 1986397], length 0
12:57:16.823051 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 1:18, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986397], length 17: HTTP: HTTP/1.0 200 OK
12:57:16.823090 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823150 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 18:155, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 137: HTTP
12:57:16.823169 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 155, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823178 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [P.], seq 155:273, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 118: HTTP
12:57:16.823184 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [.], ack 273, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823212 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [F.], seq 273, ack 79, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823266 IP 10.244.132.197.56472 > 10.244.132.196.80: Flags [F.], seq 79, ack 274, win 234, options [nop,nop,TS val 1986398 ecr 1986398], length 0
12:57:16.823272 IP 10.244.132.196.80 > 10.244.132.197.56472: Flags [.], ack 80, win 224, options [nop,nop,TS val 1986398 ecr 1986398], length 0

测试2.不同节点通过隧道进行传输

#node2 应用 访问 node1上服务

[root@ca-k8s-master01 ~]# kubectl  get pods -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
demoapp-55c5f88dcb-dj4q8   1/1     Running   0          31m   10.244.36.3      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-dpqkg   1/1     Running   0          31m   10.244.132.195   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-gwnvs   1/1     Running   0          31m   10.244.132.196   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-sxlxw   1/1     Running   0          31m   10.244.132.197   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-tmtmj   1/1     Running   0          31m   10.244.36.2      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-z4ktc   1/1     Running   0          31m   10.244.36.4      ca-k8s-node01   <none>           <none>

#通过 node1 物理网卡 ens33 进行抓包 因为进行了封装
[root@ca-k8s-node01 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
    link/ether 00:0c:29:2d:0f:c2 brd ff:ff:ff:ff:ff:ff
3: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN mode DEFAULT group default
    link/ether 02:42:7d:7f:f3:2a brd ff:ff:ff:ff:ff:ff
4: dummy0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
    link/ether 42:a7:39:13:43:9e brd ff:ff:ff:ff:ff:ff
5: kube-ipvs0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN mode DEFAULT group default
    link/ether 52:a7:b2:20:05:05 brd ff:ff:ff:ff:ff:ff
6: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
7: califc74e2139d8@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 0
8: cali55aeef544d0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 1
9: cali835b664a77d@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP mode DEFAULT group default
    link/ether ee:ee:ee:ee:ee:ee brd ff:ff:ff:ff:ff:ff link-netnsid 2

#使用node2上应用 访问 node1上应用
[root@ca-k8s-master01 ~]# kubectl exec -it demoapp-55c5f88dcb-sxlxw /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.36.3
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-dj4q8, ServerIP: 10.244.36.3!
[root@demoapp-55c5f88dcb-sxlxw /]# curl 10.244.36.3
iKubernetes demoapp v1.0 !! ClientIP: 10.244.132.197, ServerName: demoapp-55c5f88dcb-dj4q8, ServerIP: 10.244.36.3!

#node1上 查看从node2 来的报文 是通过隧道进行转发传输的
[root@ca-k8s-node01 ~]# tcpdump -i ens33 -nn host 192.168.40.122
13:08:35.237744 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [S], seq 3784827341, win 28800, options [mss 1440,sackOK,TS val 2664811 ecr 0,nop,wscale 7], length 0 (ipip-proto-4)
13:08:35.237846 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [S.], seq 3640285741, ack 3784827342, win 28560, options [mss 1440,sackOK,TS val 2657861 ecr 2664811,nop,wscale 7], length 0 (ipip-proto-4)
13:08:35.237994 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 2664812 ecr 2657861], length 0 (ipip-proto-4)
13:08:35.238064 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 2664812 ecr 2657861], length 75: HTTP: GET / HTTP/1.1 (ipip-proto-4)
13:08:35.238128 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [.], ack 76, win 224, options [nop,nop,TS val 2657861 ecr 2664812], length 0 (ipip-proto-4)
13:08:35.238880 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 2657862 ecr 2664812], length 17: HTTP: HTTP/1.0 200 OK (ipip-proto-4)
13:08:35.239003 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 2657862 ecr 2664812], length 252: HTTP (ipip-proto-4)
13:08:35.239055 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 2664813 ecr 2657862], length 0 (ipip-proto-4)
13:08:35.239192 IP 192.168.40.122 > 192.168.40.121: IP 10.244.132.197.43256 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 2664814 ecr 2657862], length 0 (ipip-proto-4)
13:08:35.239225 IP 192.168.40.121 > 192.168.40.122: IP 10.244.36.3.80 > 10.244.132.197.43256: Flags [.], ack 77, win 224, options [nop,nop,TS val 2657862 ecr 2664814], length 0 (ipip-proto-4)
13:08:35.239674 IP 192.168.40.122.10250 > 192.168.40.120.60770: Flags [P.], seq 152:302, ack 252, win 295, options [nop,nop,TS val 2664814 ecr 2631591], length 150
13:08:35.239690 IP 192.168.40.122.10250 > 192.168.40.120.60770: Flags [P.], seq 302:367, ack 252, win 295, options [nop,nop,TS val 2664814 ecr 2631591], length 65

CrossSubnet

#CrossSubnet
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml > default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# vim default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# cat default-ipv4-ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  annotations:
    projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
  creationTimestamp: "2024-01-14T16:03:32Z"
  generation: 1
  name: default-ipv4-ippool
  resourceVersion: "937"
  uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
  blockSize: 26
  cidr: 10.244.0.0/16
  ipipMode: CrossSubnet
  natOutgoing: true
  nodeSelector: all()
  vxlanMode: Never
  
[root@ca-k8s-master01 ~]# kubectl apply -f default-ipv4-ippool.yaml
Warning: resource ippools/default-ipv4-ippool is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
ippool.crd.projectcalico.org/default-ipv4-ippool configured

#确认是否修改成功
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{"projectcalico.org/metadata":"{\"uid\":\"d1166c1a-f345-4c41-beb0-86a71ee0f32d\",\"creationTimestamp\":\"2024-01-14T16:03:32Z\"}"},"creationTimestamp":"2024-01-14T16:03:32Z","generation":1,"name":"default-ipv4-ippool","resourceVersion":"937","uid":"8615a080-3692-460c-91a1-b3eb5cb8c531"},"spec":{"blockSize":26,"cidr":"10.244.0.0/16","ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()","vxlanMode":"Never"}}
    projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
  creationTimestamp: "2024-01-14T16:03:32Z"
  generation: 2
  name: default-ipv4-ippool
  resourceVersion: "8074"
  uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
  blockSize: 26
  cidr: 10.244.0.0/16
  ipipMode: CrossSubnet
  natOutgoing: true
  nodeSelector: all()
  vxlanMode: Never

#
[root@ca-k8s-master01 ~]# kubectl get pods -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
demoapp-55c5f88dcb-dj4q8   1/1     Running   0          74m   10.244.36.3      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-dpqkg   1/1     Running   0          74m   10.244.132.195   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-gwnvs   1/1     Running   0          74m   10.244.132.196   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-sxlxw   1/1     Running   0          74m   10.244.132.197   ca-k8s-node02   <none>           <none>
demoapp-55c5f88dcb-tmtmj   1/1     Running   0          74m   10.244.36.2      ca-k8s-node01   <none>           <none>
demoapp-55c5f88dcb-z4ktc   1/1     Running   0          74m   10.244.36.4      ca-k8s-node01   <none>           <none>

#查看route表 隧道turnl0 已经不用了 使用了 路由直接转发通过 ens33接口
[root@ca-k8s-master01 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.40.2    0.0.0.0         UG    100    0        0 ens33
10.244.36.0     192.168.40.121  255.255.255.192 UG    0      0        0 ens33
10.244.132.192  192.168.40.122  255.255.255.192 UG    0      0        0 ens33
10.244.237.0    0.0.0.0         255.255.255.192 U     0      0        0 *
10.244.237.3    0.0.0.0         255.255.255.255 UH    0      0        0 cali5d40fb51058
10.244.237.4    0.0.0.0         255.255.255.255 UH    0      0        0 calidadbb6c5f32
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
192.168.40.0    0.0.0.0         255.255.255.0   U     100    0        0 ens33

#通过node2 服务 访问 node1中服务 通过子网查看你
[root@ca-k8s-node01 ~]# tcpdump -i ens33 -nn net 10.244.0.0/16
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens33, link-type EN10MB (Ethernet), capture size 262144 bytes
13:44:29.479621 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [S], seq 2023735837, win 28800, options [mss 1440,sackOK,TS val 4819055 ecr 0,nop,wscale 7], length 0
13:44:29.479716 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [S.], seq 3743826456, ack 2023735838, win 28560, options [mss 1440,sackOK,TS val 4812103 ecr 4819055,nop,wscale 7], length 0
13:44:29.479863 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 4819055 ecr 4812103], length 0
13:44:29.479879 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 4819055 ecr 4812103], length 75: HTTP: GET / HTTP/1.1
13:44:29.479910 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [.], ack 76, win 224, options [nop,nop,TS val 4812103 ecr 4819055], length 0
13:44:29.480841 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 4812104 ecr 4819055], length 17: HTTP: HTTP/1.0 200 OK
13:44:29.480966 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 4812104 ecr 4819055], length 252: HTTP
13:44:29.481088 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 4819057 ecr 4812104], length 0
13:44:29.481178 IP 10.244.132.197.45000 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 4819057 ecr 4812104], length 0
13:44:29.481245 IP 10.244.36.3.80 > 10.244.132.197.45000: Flags [.], ack 77, win 224, options [nop,nop,TS val 4812105 ecr 4819057], length 0
13:44:30.307500 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [S], seq 3583458513, win 28800, options [mss 1440,sackOK,TS val 4819883 ecr 0,nop,wscale 7], length 0
13:44:30.307583 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [S.], seq 2010759158, ack 3583458514, win 28560, options [mss 1440,sackOK,TS val 4812931 ecr 4819883,nop,wscale 7], length 0
13:44:30.307749 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [.], ack 1, win 225, options [nop,nop,TS val 4819883 ecr 4812931], length 0
13:44:30.307792 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [P.], seq 1:76, ack 1, win 225, options [nop,nop,TS val 4819883 ecr 4812931], length 75: HTTP: GET / HTTP/1.1
13:44:30.307806 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [.], ack 76, win 224, options [nop,nop,TS val 4812931 ecr 4819883], length 0
13:44:30.308570 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [P.], seq 1:18, ack 76, win 224, options [nop,nop,TS val 4812932 ecr 4819883], length 17: HTTP: HTTP/1.0 200 OK
13:44:30.308699 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [FP.], seq 18:270, ack 76, win 224, options [nop,nop,TS val 4812932 ecr 4819883], length 252: HTTP
13:44:30.308754 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [.], ack 18, win 225, options [nop,nop,TS val 4819884 ecr 4812932], length 0
13:44:30.308848 IP 10.244.132.197.45010 > 10.244.36.3.80: Flags [F.], seq 76, ack 271, win 234, options [nop,nop,TS val 4819884 ecr 4812932], length 0
13:44:30.308884 IP 10.244.36.3.80 > 10.244.132.197.45010: Flags [.], ack 77, win 224, options [nop,nop,TS val 4812932 ecr 4819884], length 0

VxLan-此实验可能由于版本原因未成功 故截图

#改为:   vxlanMode: Always
[root@ca-k8s-master01 ~]# kubectl get ippools default-ipv4-ippool -o yaml > default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# vim default-ipv4-ippool.yaml
[root@ca-k8s-master01 ~]# cat default-ipv4-ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{"projectcalico.org/metadata":"{\"uid\":\"d1166c1a-f345-4c41-beb0-86a71ee0f32d\",\"creationTimestamp\":\"2024-01-14T16:03:32Z\"}"},"creationTimestamp":"2024-01-14T16:03:32Z","generation":1,"name":"default-ipv4-ippool","resourceVersion":"937","uid":"8615a080-3692-460c-91a1-b3eb5cb8c531"},"spec":{"blockSize":26,"cidr":"10.244.0.0/16","ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()","vxlanMode":"Never"}}
    projectcalico.org/metadata: '{"uid":"d1166c1a-f345-4c41-beb0-86a71ee0f32d","creationTimestamp":"2024-01-14T16:03:32Z"}'
  creationTimestamp: "2024-01-14T16:03:32Z"
  generation: 2
  name: default-ipv4-ippool
  resourceVersion: "8074"
  uid: 8615a080-3692-460c-91a1-b3eb5cb8c531
spec:
  blockSize: 26
  cidr: 10.244.0.0/16
  ipipMode: Never
  natOutgoing: true
  nodeSelector: all()
  vxlanMode: Always
  
[root@ca-k8s-master01 ~]# kubectl apply -f default-ipv4-ippool.yaml
ippool.crd.projectcalico.org/default-ipv4-ippool configured

#

 

 

Calico BGP RR配置示例


#在100个节点规模以上的Calico集群环境中,为提升iBGP的效率,通常应该建立Router Reflector。
#https://github.com/BirkhoffXia/k8s-learning/tree/master/ProjectCalico

 

posted @ 2024-02-28 13:18  しみずよしだ  阅读(121)  评论(0)    收藏  举报