容器基于OVN实现跨主机通信实验一

文档说明： 只是记录关键点

实验环境： linux debian 11
3台虚拟机
192.168.10.3 (central)
192.168.3.249 (node1)
192.168.3.250 (node2)

ovn-central 配置

#!/bin/bash

__DIR__=$(cd "$(dirname "$0")";pwd)
cd ${__DIR__}
set -uex


ovn-nbctl list dhcp_options | grep _uuid | awk '{print $3}' | xargs -i ovn-nbctl dhcp-options-del {}


ovn-nbctl --if-exists ls-del ls10
ovn-nbctl ls-add ls10


ipv4_num=$(ovn-nbctl --bare --columns=_uuid find dhcp_options cidr="10.1.20.0/24" | wc -l )

if test $ipv4_num -ne 1
then
{
    test $ipv4_num -gt 1 && ovn-nbctl --bare --columns=_uuid find dhcp_options cidr="10.1.20.0/24" | awk '{print $1}' | xargs -i ovn-nbctl dhcp-options-del {}
    ovn-nbctl dhcp-options-create "10.1.20.0/24"
}
fi
CIDR_IPV4_UUID=$(ovn-nbctl --bare --columns=_uuid find dhcp_options cidr="10.1.20.0/24")

# https://docs.openstack.org/neutron/latest/ovn/dhcp_opts.html
#server_id– 虚拟 dhcp 服务器的 ip 地址
#server_mac– 虚拟 dhcp 服务器的 MAC 地址
#lease_time– DHCP 租约的生命周期
#router键提供有关默认网关的信息

ovn-nbctl dhcp-options-set-options ${CIDR_IPV4_UUID} \
  lease_time=3600 \
  router="10.1.20.1" \
  server_id="10.1.20.1" \
  server_mac=ee:ee:02:00:00:01 \
  mtu=1400 \
  dns_server="223.5.5.5"

ovn-nbctl dhcp-options-get-options ${CIDR_IPV4_UUID}

ovn-nbctl list dhcp_options

ovn-nbctl set logical_switch ls10 \
other_config:subnet="10.1.20.0/24" \
other_config:exclude_ips="10.1.20.244..10.1.20.254"


ovn-nbctl lsp-add ls10 ls10-port2
ovn-nbctl lsp-set-addresses ls10-port2 '00:02:00:00:00:02 10.1.20.2'
ovn-nbctl lsp-set-port-security ls10-port2  '00:02:00:00:00:02 10.1.20.2'
ovn-nbctl lsp-set-dhcpv4-options ls10-port2 $CIDR_IPV4_UUID




#添加第二个 logical port
ovn-nbctl lsp-add ls10 ls10-port3
ovn-nbctl lsp-set-addresses ls10-port3 '00:02:00:00:00:03 10.1.20.3'
ovn-nbctl lsp-set-port-security ls10-port3 '00:02:00:00:00:03 10.1.20.3'
ovn-nbctl lsp-set-dhcpv4-options ls10-port3 $CIDR_IPV4_UUID

#添加第三个 logical port
ovn-nbctl lsp-add ls10 ls10-port4
ovn-nbctl lsp-set-addresses ls10-port4 '00:02:00:00:00:04 10.1.20.4'
ovn-nbctl lsp-set-port-security ls10-port4 '00:02:00:00:00:04 10.1.20.4'
ovn-nbctl lsp-set-dhcpv4-options ls10-port4 $CIDR_IPV4_UUID

ovn-nbctl list logical_switch_port
ovn-nbctl --columns dynamic_addresses list logical_switch_port
ovn-nbctl show



ovn-nbctl --if-exists lr-del lr1
ovn-nbctl lr-add lr1

ovn-nbctl lrp-add lr1 lr1-ls10-port1   ee:ee:01:00:00:01 10.1.20.1/24


ovn-nbctl lsp-add ls10 ls10-lr1-port1
ovn-nbctl lsp-set-type ls10-lr1-port1 router
ovn-nbctl lsp-set-addresses ls10-lr1-port1 router

ovn-nbctl lsp-set-options ls10-lr1-port1 router-port=lr1-ls10-port1





ovn-nbctl lrp-add lr1 lr1-public-port1   ee:ee:01:00:00:02 100.64.0.1/24



ovn-nbctl  --if-exists ls-del  public
ovn-nbctl ls-add public

ovn-nbctl lsp-add public public-lr1-port1
ovn-nbctl lsp-set-type public-lr1-port1 router
ovn-nbctl lsp-set-addresses public-lr1-port1 router
ovn-nbctl lsp-set-options public-lr1-port1 router-port=lr1-public-port1


ovn-nbctl lsp-add public public-port2
ovn-nbctl lsp-set-addresses public-port2     '00:03:00:00:00:02 100.64.0.2'
ovn-nbctl lsp-set-port-security public-port2 '00:03:00:00:00:02 100.64.0.2'

ovn-nbctl lsp-add public public-port3
ovn-nbctl lsp-set-addresses public-port3     '00:03:00:00:00:03 100.64.0.3'
ovn-nbctl lsp-set-port-security public-port3 '00:03:00:00:00:03 100.64.0.3'



ovn-nbctl --policy=dst-ip lr-route-add lr1 "0.0.0.0/0" 100.64.0.1

ovn-nbctl lr-policy-add lr1 32767 "ip4.dst == 10.1.20.0/24"   allow
ovn-nbctl lr-policy-add lr1 32767 "ip4.dst == 100.64.0.0/16"  allow

ovn-nbctl lr-policy-add lr1 30000 "ip4.dst == 192.168.3.250" reroute 100.64.0.3
ovn-nbctl lr-policy-add lr1 30000 "ip4.dst == 192.168.3.249" reroute 100.64.0.2

ovn-nbctl lr-policy-add lr1 29990 "ip4.src == 10.1.20.0/24"  reroute  100.64.0.3

# lr-policy-add ROUTER PRIORITY MATCH ACTION [NEXTHOP]
# https://www.ovn.org/support/dist-docs/ovn-nbctl.8.txt
# https://www.ovn.org/support/dist-docs/

ovn-nbctl lr-policy-list lr1
ovn-nbctl lr-route-list lr1
ovn-nbctl lr-nat-list lr1 
ovn-nbctl lr-lb-list lr1

节点 192.168.3.249

#!/bin/bash
set -uex


ovs_running_flag=$(ps -ef | grep 'ovs-vswitchd unix:/usr/local/var/run/openvswitch/db.sock' | grep -v 'grep')

if test -z "$ovs_running_flag"
  then
    echo 'ovs no running' && exit 1
fi

set -ux
# grep命令精确匹配字符串查找

flag=$(ip netns list | grep "\<vm1\>")
test -z "$flag"  || ip netns del vm1

ip netns add vm1

ovs-vsctl --if-exists del-port br-int vm1
ovs-vsctl --may-exist add-port br-int vm1 -- set interface vm1 type=internal -- set Interface vm1 external_ids:iface-id=ls10-port2

ip link set vm1 netns vm1

ip netns exec vm1 ip link set vm1 address 00:02:00:00:00:02
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip link set lo up
ip netns exec vm1 dhclient -v
ip netns exec vm1 ip a




#!/bin/bash
set -uex

ovs-vsctl --if-exists del-port  br-int  ovn0
ovs-vsctl add-port  br-int  ovn0  -- \
          set interface ovn0  type=internal -- \
          set interface ovn0  external_ids:iface-id=public-port2 -- \
          set interface ovn0  external_ids:ip=100.64.0.2

ip link set dev  ovn0 up
ip link set dev ovn0 mtu 1400
ip link set dev ovn0 address 00:03:00:00:00:02
ip addr add 100.64.0.2/24 dev ovn0

ip route add 10.1.20.0/24 via 100.64.0.1
ip route add 100.64.0.0/24 via 100.64.0.1

节点 192.168.3.250

#!/bin/bash
set -uex 

ovs_running_flag=$(ps -ef | grep 'ovs-vswitchd unix:/usr/local/var/run/openvswitch/db.sock' | grep -v 'grep')

if test -z "$ovs_running_flag"
  then
    echo 'ovs no running' && exit 1
fi

set -ux
# grep命令精确匹配字符串查找

flag=$(ip netns list | grep "\<vm1\>")
test -z "$flag"  || ip netns del vm1

ip netns add vm1

ovs-vsctl --if-exists del-port br-int vm1
ovs-vsctl --may-exist add-port br-int vm1 -- set interface vm1 type=internal -- set Interface vm1 external_ids:iface-id=ls10-port3

ip link set vm1 netns vm1

ip netns exec vm1 ip link set vm1 address 00:02:00:00:00:03
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip link set lo up
ip netns exec vm1 dhclient -v
ip netns exec vm1 ip a

#ip link set mtu 1450 dev br-provider
#ovs-vsctl set int br-int mtu_request=1450


#!/bin/bash
set -uex 
ovs-vsctl --if-exists del-port  br-int  ovn0
ovs-vsctl add-port  br-int  ovn0  -- \
          set interface ovn0  type=internal -- \
          set interface ovn0  external_ids:iface-id=public-port3 -- \
          set interface ovn0  external_ids:ip=100.64.0.3

ip link set dev  ovn0 up
ip addr add 100.64.0.3/24 dev ovn0
ip link set dev ovn0 mtu 1400
ip link set dev ovn0 address 00:03:00:00:00:03

ip route add 10.1.20.0/24 via 100.64.0.1
ip route add 100.64.0.0/24 via 100.64.0.1

iptables -t nat -A POSTROUTING -s 10.1.20.0/24 -o enp0s3 -j MASQUERADE

工具

ip route show
route -n
netstat -nr

iptables -t nat -L  -n --line-number

tcpdump -i any   port 6081 -v

ethtool 

tcpdump -i any   port 6081 -v -n
apt install -y conntrack

# 跟踪它看到的所有报文流
conntrack -L
 # 可显示经过源 NAT 的连接跟踪项
conntrack -L -p tcp –src-nat
tcpdump -i any   not host 192.168.10.3 and not host 192.168.3.26 -v -n

参考文档一：

  1. 单网卡加入OVS网桥

  2. OVN虚拟网络出网网关配置

  3. OVN路由器对等连接

  4. KUBE-OVN如何实现POD和主机网络连通

  5. OVN路由器对等连接

  6. ovn通过宿主机出网方案

  7. how-to-create-an-open-virtual-network-distributed-gateway-router

  8. Dynamic IP address management in Open Virtual Network (OVN): Part Two

  9. SNAT和DNAT简介

  10. ovn-gateway-practice

  11. ovn DHCP

  12. iptables四表五链

  13. ovs-docker-ovn.sh

  14. SDN网络指南

  15. Overlay-Geneve

  16. ovn 通过分布式网关端口连接外部网络

  17. ovn 通过网关虚拟路由器连接外部网络

  18. ovn 配置逻辑路由器实现三层转发

  19. ovn创建vpc 的snat 出外网

  20. how-to-create-an-open-virtual-network-distributed-gateway-router

参考文档二: ovn-central :

  1. OVN介绍及安装流程
  2. 如何配置OVN路由器
  3. 如何将OVN虚拟网络连接到外部网络
  4. 如何配置OVN负载均衡器？

参考文档三：

  1. Anycast概述
  2. 互联网网间互联方式，什么是对等互联？
  3. Underlay、Overlay、大二层介绍
  4. 未来网络白皮书——白盒交换机技术白皮书.pdf
  5. OVN-IC例子
  6. ovn-InterConnection
  7. ovn为外部主机提供dhcp服务
  8. OVN路由器对等连接
  9. An introduction to Linux virtual interfaces: Tunnels
  10. 时间敏感网络交换机 TSN switch
  11. VoIP, VoLTE, VoNR 与 IMS 的联系
  12. BFD（Bidirectional Forwarding Detection，双向转发检测）
  13. ECMP （等价路由） 多路径负载均衡和链路备份的目的
  14. 分段路由 SRv6
  15. 理解Segment Routing和SDWAN
  16. Geneve（Generic Network Virtualization Encapsulation） 通用的封装协议标准
  17. 一文总结 Linux 虚拟网络设备 eth, tap/tun, veth-pair
  18. 开源治理白皮书
  19. 生成自签名的SSL证书

官方文档:

  1. ovn-dist-docs
  2. OVS-dist-docs-2.5
  3. ovs-latest-contents
  4. ovs faq
  5. OVN-Tutorial
  6. ovn
  7. ovn-ref
  8. ovn-ipsec
  9. ovn-dist-docs
  10. ovn-interconnection
  11. Open vSwitch with KVM
  12. Using Open vSwitch with DPDK
  13. Open vSwitch with SSL
  14. Multi-tenant Inter-DC tunneling with OVN

上一篇 OVN启动 ovn-central 和 ovn-controller

下一篇容器基于OVN实现跨主机通信实验二(正在实验中)

posted @ 2022-10-22 14:45  jingjingxyk  阅读(553)  评论(0)  编辑  收藏  举报