ovs流表机制（四）用vxlan实现不同节点的vm通信--流表learn
10.10.16.82节点上
[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.22 PING 192.168.10.22 (192.168.10.22) 56(84) bytes of data. ^C --- 192.168.10.22 ping statistics --- 62 packets transmitted, 0 received, 100% packet loss, time 63406ms
10.10.16.82节点上
[root@kunpeng82 devuser]# tcpdump -i vxlan_sys_4789 -nnvv tcpdump: listening on vxlan_sys_4789, link-type EN10MB (Ethernet), capture size 262144 bytes 13:05:18.307159 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:19.347186 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:20.387152 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:21.427150 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:22.467182 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:23.507152 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 13:05:24.547151 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 ^C 7 packets captured 7 packets received by filter 0 packets dropped by kernel
10.10.18.216节点上
[root@host-10-10-18-216 ~]# tcpdump -i vxlan_sys_4789 -nnvv tcpdump: listening on vxlan_sys_4789, link-type EN10MB (Ethernet), capture size 262144 bytes 21:05:38.323613 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:39.363582 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:40.403603 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:41.443641 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:42.483594 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:43.523588 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:44.563627 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:45.603574 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:46.643548 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:47.683565 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:48.723543 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:49.763509 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 21:05:50.803536 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 ^C
br-tun也抓不到报文，看来ovs和linux br还是有区别哦
[root@host-10-10-18-216 ~]# tcpdump -i br-tun -nnvv tcpdump: listening on br-tun, link-type EN10MB (Ethernet), capture size 262144 bytes ^C 0 packets captured 0 packets received by filter 0 packets dropped by kernel
原来是br-tun的流表被删除了，加上之后就可以了
[root@host-10-10-18-216 ~]# ovs-ofctl dump-flows br-tun [root@host-10-10-18-216 ~]# ovs-ofctl dump-flows br-int cookie=0x0, duration=180739.746s, table=0, n_packets=200, n_bytes=14424, priority=0 actions=NORMAL [root@host-10-10-18-216 ~]# ovs-ofctl add-flow br-tun "priority=0,actions=NORMAL" [root@host-10-10-18-216 ~]# ovs-ofctl dump-flows br-tun cookie=0x0, duration=6.329s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL
可以ping通了
[root@kunpeng82 devuser]# ip netns exec ns1 ping 192.168.10.22 PING 192.168.10.22 (192.168.10.22) 56(84) bytes of data. 64 bytes from 192.168.10.22: icmp_seq=1 ttl=64 time=1.65 ms 64 bytes from 192.168.10.22: icmp_seq=2 ttl=64 time=0.266 ms 64 bytes from 192.168.10.22: icmp_seq=3 ttl=64 time=0.734 ms 64 bytes from 192.168.10.22: icmp_seq=4 ttl=64 time=0.295 ms ^C --- 192.168.10.22 ping statistics --- 4 packets transmitted, 4 received, 0% packet loss, time 3118ms rtt min/avg/max/mdev = 0.266/0.736/1.650/0.559 ms [root@kunpeng82 devuser]#
查看fdb表
[root@host-10-10-18-216 ~]# ovs-appctl fdb/show br-int port VLAN MAC Age 2 22 7e:6c:77:3c:99:49 266 1 22 2e:6b:9b:c1:f9:2d 46 2 22 7e:53:2a:70:ac:48 46 1 22 c2:31:85:2b:fa:02 11 [root@host-10-10-18-216 ~]# ovs-appctl fdb/show br-tun port VLAN MAC Age 1 22 7e:6c:77:3c:99:49 273 2 22 2e:6b:9b:c1:f9:2d 53 1 22 7e:53:2a:70:ac:48 53 2 22 c2:31:85:2b:fa:02 18
把qvo1 改为tag=100
先trace一下
[root@host-10-10-18-216 ~]# ovs-appctl ofproto/trace br-int in_port=2,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,ip,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_proto=1 -generate Flow: icmp,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0 bridge("br-int") ---------------- 0. priority 0 NORMAL -> learned that 7e:53:2a:70:ac:48 is on port qvo1 in VLAN 100 -> no learned MAC for destination, flooding bridge("br-tun") ---------------- 0. in_port=1, priority 1, cookie 0x79 resubmit(,2) 2. dl_dst=00:00:00:00:00:00/01:00:00:00:00:00, priority 0, cookie 0x79 resubmit(,20) 20. priority 0, cookie 0x79 resubmit(,22) 22. dl_vlan=100, priority 32768, cookie 0x79 // 未知单播 strip_vlan set_tunnel:0x20 output:2 -> output to kernel tunnel Final flow: unchanged Megaflow: recirc_id=0,eth,ip,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_ecn=0,nw_frag=no Datapath actions: push_vlan(vid=100,pcp=0),5,set(tunnel(tun_id=0x10,dst=10.10.16.82,ttl=64,tp_dst=4789,flags(df|key))),pop_vlan,3
10.10.16.82节点抓包
[root@kunpeng82 devuser]# tcpdump -i enahisic2i0 host 10.10.18.216 -nnvv tcpdump: listening on enahisic2i0, link-type EN10MB (Ethernet), capture size 262144 bytes 04:51:14.852881 IP (tos 0x0, ttl 63, id 5163, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 04:51:15.892893 IP (tos 0x0, ttl 63, id 5256, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 04:51:16.933037 IP (tos 0x0, ttl 63, id 5274, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 04:51:17.972920 IP (tos 0x0, ttl 63, id 5307, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 04:51:19.012903 IP (tos 0x0, ttl 63, id 5333, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 04:51:20.053042 IP (tos 0x0, ttl 63, id 5353, offset 0, flags [DF], proto UDP (17), length 78) 10.10.18.216.36528 > 10.10.16.82.4789: [no cksum] VXLAN, flags [I] (0x08), vni 16 ARP, Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 ^C
10.10.16.82节点trace
[root@kunpeng82 devuser]# ovs-appctl ofproto/trace br-tun in_port=2,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,ip,tun_id=0x10,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_proto=1 -generate Flow: icmp,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0 bridge("br-tun") ---------------- 0. in_port=2, priority 1, cookie 0x79 resubmit(,4) 4. priority 0, cookie 0x79 drop Final flow: unchanged Megaflow: recirc_id=0,eth,ip,tun_id=0,in_port=2,nw_frag=no Datapath actions: drop
原来把vxlan的id搞错了
cookie=0x79, duration=485.164s, table=4, n_packets=0, n_bytes=0, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10) cookie=0x79, duration=485.158s, table=4, n_packets=94, n_bytes=3976, priority=0 actions=drop
Port "vxlan-01" Interface "vxlan-01" type: vxlan options: {key="0x10", remote_ip="10.10.18.216"}
[root@kunpeng82 devuser]# ovs-ofctl del-flows br-tun table=4 [root@kunpeng82 devuser]# ovs-ofctl dump-flows br-tun table=4
[root@kunpeng82 devuser]# ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10)'
[root@kunpeng82 devuser]# ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=0 actions=drop'
[root@kunpeng82 devuser]#
[root@kunpeng82 devuser]# ovs-appctl ofproto/trace br-tun in_port=2,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,ip,tun_id=0X10,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_proto=1 -generate
Flow: icmp,tun_id=0x20,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
bridge("br-tun")
----------------
0. in_port=2, priority 1, cookie 0x79
resubmit(,4)
4. tun_id=0x10, priority 1, cookie 0x79
mod_vlan_vid:22
resubmit(,10)
10. priority 1, cookie 0x79
learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[])
-> table=20 vlan_tci=0x0016/0x0fff,dl_dst=7e:53:2a:70:ac:48 priority=1 cookie=0xa9eb8f9011f7e038 hard=300 actions=load:0->NXM_OF_VLAN_TCI[],load:0x10->NXM_NX_TUN_ID[],output:2
output:1
bridge("br-int")
----------------
0. priority 0
NORMAL
-> learned that 7e:53:2a:70:ac:48 is on port patch-tun in VLAN 22
-> no learned MAC for destination, flooding
Final flow: icmp,tun_id=0x20,in_port=2,dl_vlan=22,dl_vlan_pcp=0,vlan_tci1=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_src=192.168.10.12,nw_dst=192.168.10.22,nw_tos=0,nw_ecn=0,nw_ttl=0,icmp_type=0,icmp_code=0
Megaflow: recirc_id=0,eth,ip,tun_id=0x10,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=2e:6b:9b:c1:f9:2d,nw_frag=no
Datapath actions: push_vlan(vid=22,pcp=0),2,pop_vlan,3
但是ping还是不通
[root@kunpeng82 devuser]# tcpdump -i vxlan_sys_4789 -eennvv tcpdump: listening on vxlan_sys_4789, link-type EN10MB (Ethernet), capture size 262144 bytes 07:25:52.580006 7e:53:2a:70:ac:48 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 07:25:53.620035 7e:53:2a:70:ac:48 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 07:25:54.660151 7e:53:2a:70:ac:48 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 07:25:55.700060 7e:53:2a:70:ac:48 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 ^C 4 packets captured 4 packets received by filter 0 packets dropped by kernel
arp trace
[root@kunpeng82 devuser]# ovs-appctl ofproto/trace br-tun in_port=2,dl_src=7e:53:2a:70:ac:48,dl_dst=ff:ff:ff:ff:ff:ff,arp,tun_id=0X20 -generate Flow: arp,tun_id=0x20,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=0.0.0.0,arp_tpa=0.0.0.0,arp_op=0,arp_sha=00:00:00:00:00:00,arp_tha=00:00:00:00:00:00 bridge("br-tun") ---------------- 0. in_port=2, priority 1, cookie 0x79 resubmit(,4) 4. tun_id=0x20, priority 1, cookie 0x79 mod_vlan_vid:22 resubmit(,10) 10. priority 1, cookie 0x79 learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]) -> table=20 vlan_tci=0x0016/0x0fff,dl_dst=7e:53:2a:70:ac:48 priority=1 cookie=0xa9eb8f9011f7e038 hard=300 actions=load:0->NXM_OF_VLAN_TCI[],load:0x20->NXM_NX_TUN_ID[],output:2 output:1 bridge("br-int") ---------------- 0. priority 0 NORMAL -> learned that 7e:53:2a:70:ac:48 is on port patch-tun in VLAN 22 -> no learned MAC for destination, flooding Final flow: arp,tun_id=0x20,in_port=2,dl_vlan=22,dl_vlan_pcp=0,vlan_tci1=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=0.0.0.0,arp_tpa=0.0.0.0,arp_op=0,arp_sha=00:00:00:00:00:00,arp_tha=00:00:00:00:00:00 Megaflow: recirc_id=0,eth,arp,tun_id=0x20,in_port=2,vlan_tci=0x0000,dl_src=7e:53:2a:70:ac:48,dl_dst=ff:ff:ff:ff:ff:ff,arp_op=0 Datapath actions: push_vlan(vid=22,pcp=0),2,pop_vlan,3
10.10.18.216上查看下vxlan id,发现vxlan id是0x10
[root@host-10-10-18-216 ~]# ovs-vsctl show 0608fdfe-ab3f-41d8-a439-b0dba68f452b Bridge br-tun fail_mode: secure Port br-tun Interface br-tun type: internal Port patch-int Interface patch-int type: patch options: {peer=patch-tun} Port "vxlan-01" Interface "vxlan-01" type: vxlan options: {key="0x10", remote_ip="10.10.16.82"}
[root@host-10-10-18-216 ~]# ovs-vsctl del-port vxlan-01
[root@host-10-10-18-216 ~]# ovs-vsctl add-port br-tun vxlan-01 -- set interface vxlan-01 type=vxlan options:remote_ip=10.10.16.82 options:key=0x20 ofport_request=2
最后可以ping通了
[root@host-10-10-18-216 ~]# ip netns exec ns1 ping 192.168.10.12
PING 192.168.10.12 (192.168.10.12) 56(84) bytes of data.
64 bytes from 192.168.10.12: icmp_seq=1 ttl=64 time=2.05 ms
64 bytes from 192.168.10.12: icmp_seq=2 ttl=64 time=0.293 ms
^C
--- 192.168.10.12 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1001ms
rtt min/avg/max/mdev = 0.293/1.174/2.056/0.882 ms
[root@host-10-10-18-216 ~]#
查看table 20
[root@kunpeng82 devuser]# ovs-ofctl dump-flows br-tun table=20 cookie=0xa9eb8f9011f7e038, duration=7.494s, table=20, n_packets=4, n_bytes=280, hard_timeout=300, priority=1,vlan_tci=0x0016/0x0fff,dl_dst=7e:53:2a:70:ac:48 actions=load:0->NXM_OF_VLAN_TCI[],load:0x20->NXM_NX_TUN_ID[],output:"vxlan-01" cookie=0x79, duration=6670.529s, table=20, n_packets=0, n_bytes=0, priority=0 actions=resubmit(,22) [root@kunpeng82 devuser]#
[root@host-10-10-18-216 ~]# ovs-ofctl dump-flows br-tun table=20
cookie=0xa9eb8f9011f7e038, duration=24.363s, table=20, n_packets=3, n_bytes=182, hard_timeout=300, priority=1,vlan_tci=0x0064/0x0fff（本地vlan tag）,dl_dst=2e:6b:9b:c1:f9:2d(目的mac) actions=load:0->NXM_OF_VLAN_TCI[],load:0x20->NXM_NX_TUN_ID[],output:"vxlan-01"
cookie=0x79, duration=12530.579s, table=20, n_packets=12, n_bytes=1008, priority=0 actions=resubmit(,22)
[root@host-10-10-18-216 ~]#
查看neigh
[root@host-10-10-18-216 ~]# ip netns exec ns1 ip n 192.168.10.12 dev tap0 FAILED 192.168.1.200 dev tap0 lladdr 8a:6a:5f:bb:b0:9a STALE 192.168.10.12 dev veth1 lladdr 2e:6b:9b:c1:f9:2d STALE 114.114.114.114 dev tap0 FAILED [root@host-10-10-18-216 ~]#
bridge fdb show
[root@host-10-10-18-216 ~]# bridge fdb show 01:00:5e:00:00:01 dev eth0 self permanent 33:33:00:00:00:01 dev eth0 self permanent 33:33:ff:0a:1e:35 dev eth0 self permanent 01:00:5e:00:00:fb dev eth0 self permanent 33:33:00:00:00:01 dev tap0_br self permanent 01:00:5e:00:00:01 dev tap0_br self permanent 33:33:ff:c3:35:36 dev tap0_br self permanent 33:33:00:00:00:01 dev ovs-system self permanent 33:33:00:00:00:01 dev vswitch0 self permanent 01:00:5e:00:00:01 dev vswitch0 self permanent 33:33:ff:cc:11:49 dev vswitch0 self permanent 01:00:5e:00:00:01 dev br-tun self permanent 33:33:00:00:00:01 dev br-tun self permanent 33:33:ff:3e:ff:41 dev br-tun self permanent 33:33:00:00:00:01 dev br-int self permanent 01:00:5e:00:00:01 dev br-int self permanent 33:33:ff:fb:40:48 dev br-int self permanent 33:33:00:00:00:01 dev qbr1 self permanent 01:00:5e:00:00:01 dev qbr1 self permanent 33:33:ff:76:5e:c3 dev qbr1 self permanent 7e:6c:77:3c:99:49 dev qvb1 vlan 1 master qbr1 permanent 7e:6c:77:3c:99:49 dev qvb1 master qbr1 permanent 33:33:00:00:00:01 dev qvb1 self permanent 01:00:5e:00:00:01 dev qvb1 self permanent 33:33:ff:3c:99:49 dev qvb1 self permanent 33:33:00:00:00:01 dev qvo1 self permanent 01:00:5e:00:00:01 dev qvo1 self permanent 33:33:ff:5e:18:a1 dev qvo1 self permanent fa:56:81:d9:2c:e2 dev veth1_br master qbr1 permanent fa:56:81:d9:2c:e2 dev veth1_br vlan 1 master qbr1 permanent 33:33:00:00:00:01 dev veth1_br self permanent 01:00:5e:00:00:01 dev veth1_br self permanent 33:33:ff:d9:2c:e2 dev veth1_br self permanent
ovs-appctl fdb/show
[root@host-10-10-18-216 ~]# ovs-appctl fdb/show br-int port VLAN MAC Age 2 100 7e:6c:77:3c:99:49 145 1 100 2e:6b:9b:c1:f9:2d 5 2 100 7e:53:2a:70:ac:48 5 [root@host-10-10-18-216 ~]# ovs-appctl fdb/show br-tun port VLAN MAC Age -----------------竟然是空的哦 [root@host-10-10-18-216 ~]#
ovs-appctl fdb/show
[root@kunpeng82 devuser]# ovs-appctl fdb/show br-tun
port VLAN MAC Age
[root@kunpeng82 devuser]# ovs-appctl fdb/show br-int
port VLAN MAC Age
1 22 7e:6c:77:3c:99:49 191
1 22 7e:53:2a:70:ac:48 50
2 22 2e:6b:9b:c1:f9:2d 50
[root@kunpeng82 devuser]#
现在删除192.168.10.12的neighbor项，看10.10.16.82能不能收到arp广播
[root@host-10-10-18-216 ~]# ip netns exec ns1 ip n del 192.168.10.12 dev veth1 lladdr 2e:6b:9b:c1:f9:2d [root@host-10-10-18-216 ~]# ip netns exec ns1 ping 192.168.10.12 PING 192.168.10.12 (192.168.10.12) 56(84) bytes of data. 64 bytes from 192.168.10.12: icmp_seq=1 ttl=64 time=1.55 ms 64 bytes from 192.168.10.12: icmp_seq=2 ttl=64 time=0.290 ms 64 bytes from 192.168.10.12: icmp_seq=3 ttl=64 time=0.273 ms ^C --- 192.168.10.12 ping statistics --- 3 packets transmitted, 3 received, 0% packet loss, time 2008ms rtt min/avg/max/mdev = 0.273/0.704/1.550/0.598 ms [root@host-10-10-18-216 ~]#
[root@kunpeng82 devuser]# ip netns exec ns1 tcpdump -i veth1 arp -eennvv tcpdump: listening on veth1, link-type EN10MB (Ethernet), capture size 262144 bytes 08:19:09.015788 7e:53:2a:70:ac:48 > ff:ff:ff:ff:ff:ff, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.12 tell 192.168.10.22, length 28 08:19:09.015796 2e:6b:9b:c1:f9:2d > 7e:53:2a:70:ac:48, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Reply 192.168.10.12 is-at 2e:6b:9b:c1:f9:2d, length 28 08:19:14.067128 2e:6b:9b:c1:f9:2d > 7e:53:2a:70:ac:48, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 192.168.10.22 tell 192.168.10.12, length 28 08:19:14.067659 7e:53:2a:70:ac:48 > 2e:6b:9b:c1:f9:2d, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Reply 192.168.10.22 is-at 7e:53:2a:70:ac:48, length 28 ^C 4 packets captured 4 packets received by filter 0 packets dropped by kernel [root@kunpeng82 devuser]#
10.10.16.82节点配置---流表
## patch-int port =1 ## vxlan-01 port =2 ##================== table 0 #处理patch-int包，虚机出流量 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=1,in_port=1 actions=resubmit(,2)' #处理vxlan包，从vxlan tunnel进入的数据包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=1,in_port=2 actions=resubmit(,4)' #默认规则 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=0 actions=drop' #处理patch-int包，虚机出流量 table 0 --> resubmit 2 #===========table 2 #单播包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=2, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)' #广播包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=2, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)' #=================== table 3 ovs-ofctl add-flow br-tun 'cookie=0x79, table=3, priority=0 actions=drop' #处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4) #vxlan id =0x20, 10.10.16.82节点 vlan id =22 #=================== table 4
// 把vxlan映射到对应的vlan ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=1,tun_id=0x20 actions=mod_vlan_vid:22,resubmit(,10)' ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=0 actions=drop' #=================== table 6 ovs-ofctl add-flow br-tun 'cookie=0x79, table=6, priority=0 actions=drop' #处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4) # table 4 mod_vlan_id --> resubmit(10) #==================== table 10 ovs-ofctl add-flow br-tun 'cookie=0x79, table=10, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1' ## table 10 learn --> table 20 saves learning resuslt #处理patch-int包，虚机出流量 table 0 --> resubmit 2 # table 2 unicast --> table 20 ##===================== table 20 没有match则跳转到 table 22 # table 20 ovs-ofctl add-flow br-tun 'cookie=0x79, table=20, priority=0 actions=resubmit(,22)' #处理patch-int包，虚机出流量 vlan tag=22, table 0 --> resubmit 2 # table 2 broadcast --> table 22 ##======================== table 22 flood
//把vlan 映射到对应的vxlan,如果没有strip_vlan则vxlan承载的mac报文包含vlan头 ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=22 actions=strip_vlan,set_tunnel:0x20,output:2' ovs-ofctl add-flow br-tun 'cookie=0x79, table=22,priority=0 actions=drop'
设置vxlan端口为trunk类型并配置可以转发的vlan
ovs-vsctl set Port vxlan trunks=100,200
10.10.16.82节点配置--网络
#/bin/bash set -e #添加ovs网桥: br-tun/br-int #ovs-vsctl add-br br-tun #ovs-vsctl add-br br-int #ovs-vsctl set-fail-mode br-tun secure #ovs-vsctl set-fail-mode br-int standalone #添加patch口，连接br-tun,br-int #ovs-vsctl add-port br-tun patch-int -- set interface patch-int type=patch -- set interface patch-int options:peer=patch-tun #ovs-vsctl add-port br-int patch-tun -- set interface patch-tun type=patch -- set interface patch-tun options:peer=patch-int #添加linux bridge #brctl addbr qbr1 #ip link set qbr1 up #添加veth口，连接br-int,qbr1 #ip link add name qvo1 type veth peer name qvb1 #ip link set qvo1 up #ip link set qvb1 up #ovs-vsctl add-port br-int qvo1 #brctl addif qbr1 qvb1 ##设置qvo1的vlan tag #ovs-vsctl set port qvo1 tag=22 #添加namespace： #ip netns add ns1 #ip link add name veth1 type veth peer name veth1_br #ip link set dev veth1 netns ns1 #brctl addif qbr1 veth1_br #ip link set veth1_br up #ip netns exec ns1 ip link set veth1 up #ip netns exec ns1 ip addr add 192.168.10.12/24 dev veth1 ovs-vsctl del-port vxlan-01 ovs-vsctl add-port br-tun vxlan-01 -- set interface vxlan-01 type=vxlan options:remote_ip=10.10.18.216 options:key=0x20
10.10.18.216节点配置--流表
## patch-int port =1 ## vxlan-01 port =2 ##================== table 0 #处理patch-int包，虚机出流量 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=1,in_port=1 actions=resubmit(,2)' #处理vxlan包，从vxlan tunnel进入的数据包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=1,in_port=2 actions=resubmit(,4)' #默认规则 ovs-ofctl add-flow br-tun 'cookie=0x79, table=0, priority=0 actions=drop' #处理patch-int包，虚机出流量 table 0 --> resubmit 2 #===========table 2 #单播包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=2, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20)' #广播包 ovs-ofctl add-flow br-tun 'cookie=0x79, table=2, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22)' #=================== table 3 ovs-ofctl add-flow br-tun 'cookie=0x79, table=3, priority=0 actions=drop' #处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4) #vxlan id =0x20, 10.10.18.216节点 vlan id =100 #=================== table 4 ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=1,tun_id=0x20 actions=mod_vlan_vid:100,resubmit(,10)' ovs-ofctl add-flow br-tun 'cookie=0x79, table=4, priority=0 actions=drop' #=================== table 6 ovs-ofctl add-flow br-tun 'cookie=0x79, table=6, priority=0 actions=drop' #处理vxlan包，从vxlan tunnel进入的数据包会resubmit(4) # table 4 mod_vlan_id --> resubmit(10) #==================== table 10 ovs-ofctl add-flow br-tun 'cookie=0x79, table=10, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:1' ## table 10 learn --> table 20 saves learning resuslt #处理patch-int包，虚机出流量 table 0 --> resubmit 2 # table 2 unicast --> table 20 ##===================== table 20 没有match(uncast)则跳转到 table 22 # table 20 ovs-ofctl add-flow br-tun 'cookie=0x79, table=20, priority=0 actions=resubmit(,22)' #处理patch-int包，虚机出流量 table 0 --> resubmit 2 # table 2 broadcast --> table 22 ##======================== table 22 flood ovs-ofctl add-flow br-tun 'cookie=0x79, table=22, dl_vlan=100 actions=strip_vlan,set_tunnel:0x20,output:2' ovs-ofctl add-flow br-tun 'cookie=0x79, table=22,priority=0 actions=drop'
10.10.18.216节点配置--网络
/bin/bash et -e 添加ovs网桥: br-tun/br-int ovs-vsctl add-br br-tun ovs-vsctl add-br br-int ovs-vsctl set-fail-mode br-tun secure ovs-vsctl set-fail-mode br-int standalone 添加patch口，连接br-tun,br-int ovs-vsctl add-port br-tun patch-int -- set interface patch-int type=patch -- set interface patch-int options:peer=patch-tun ovs-vsctl add-port br-int patch-tun -- set interface patch-tun type=patch -- set interface patch-tun options:peer=patch-int 添加linux bridge brctl addbr qbr1 ip link set qbr1 up 添加veth口，连接br-int,qbr1 ip link add name qvo1 type veth peer name qvb1 ip link set qvo1 up ip link set qvb1 up ovs-vsctl add-port br-int qvo1 brctl addif qbr1 qvb1 #设置qvo1的vlan tag ovs-vsctl set port qvo1 tag=100 添加namespace： ip netns add ns1 ip link add name veth1 type veth peer name veth1_br ip link set dev veth1 netns ns1 brctl addif qbr1 veth1_br ip link set veth1_br up ip netns exec ns1 ip link set veth1 up ip netns exec ns1 ip addr add 192.168.10.22/24 dev veth1 ovs-vsctl del-port vxlan-01 ovs-vsctl add-port br-tun vxlan-01 -- set interface vxlan-01 type=vxlan options:remote_ip=10.10.16.82 options:key=0x20 ofport_request=2 # ovs-vsctl del-port vxlan-01
[root@host-10-10-18-216 ~]# ovs-vsctl dump-flows br-tun ovs-vsctl: unknown command 'dump-flows'; use --help for help [root@host-10-10-18-216 ~]# ovs-ofctl dump-flows br-tun cookie=0x79, duration=31570.269s, table=0, n_packets=1916, n_bytes=82768, priority=1,in_port="patch-int" actions=resubmit(,2) cookie=0x79, duration=31570.265s, table=0, n_packets=427, n_bytes=19278, priority=1,in_port="vxlan-01" actions=resubmit(,4) cookie=0x79, duration=31570.261s, table=0, n_packets=9, n_bytes=630, priority=0 actions=drop cookie=0x79, duration=31570.256s, table=2, n_packets=34, n_bytes=2772, priority=0,dl_dst=00:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,20) cookie=0x79, duration=31570.251s, table=2, n_packets=1882, n_bytes=79996, priority=0,dl_dst=01:00:00:00:00:00/01:00:00:00:00:00 actions=resubmit(,22) cookie=0x79, duration=31570.247s, table=3, n_packets=0, n_bytes=0, priority=0 actions=drop cookie=0x79, duration=31570.242s, table=4, n_packets=42, n_bytes=3108, priority=1,tun_id=0x20 actions=mod_vlan_vid:100,resubmit(,10) cookie=0x79, duration=31570.238s, table=4, n_packets=385, n_bytes=16170, priority=0 actions=drop cookie=0x79, duration=31570.233s, table=6, n_packets=0, n_bytes=0, priority=0 actions=drop cookie=0x79, duration=31570.229s, table=10, n_packets=42, n_bytes=3108, priority=1 actions=learn(table=20,hard_timeout=300,priority=1,cookie=0xa9eb8f9011f7e038,NXM_OF_VLAN_TCI[0..11],NXM_OF_ETH_DST[]=NXM_OF_ETH_SRC[],load:0->NXM_OF_VLAN_TCI[],load:NXM_NX_TUN_ID[]->NXM_NX_TUN_ID[],output:NXM_OF_IN_PORT[]),output:"patch-int" cookie=0x79, duration=31570.224s, table=20, n_packets=13, n_bytes=1106, priority=0 actions=resubmit(,22) cookie=0x79, duration=31570.220s, table=22, n_packets=1887, n_bytes=80542, dl_vlan=100 actions=strip_vlan,set_tunnel:0x20,output:"vxlan-01" cookie=0x79, duration=31570.216s, table=22, n_packets=8, n_bytes=560, priority=0 actions=drop
