# 网络calico
## BGP进阶
```sh
cd /usr/local/bin/
curl -o calicoctl -O -L https://github.com/projectcalico/calicoctl/releases/download/v3.20.2/calicoctl
[00:41:57 root@master1 bin]#chmod +x calicoctl
[22:39:36 root@master1 calico]#cat /etc/calico/calicoctl.cfg
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
datastoreType: "kubernetes"
kubeconfig: "/etc/kubernetes/admin.conf"
# 测试
[22:40:04 root@master1 calico]#calicoctl get nodes
NAME
master1
master2.noisedu.cn
master3.noisedu.cn
node1.noisedu.cn
node2.noisedu.cn
# 关联kubectl
[22:42:05 root@master1 calico]#cd /usr/local/bin/
[22:42:25 root@master1 bin]#cp -p calicoctl kubectl-calico
[00:43:00 root@master1 bin]#kubectl calico node status
Calico process is running.
IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+--------------+-------------------+-------+----------+-------------+
| 10.0.0.51 | node-to-node mesh | up | 16:41:21 | Established |
| 10.0.0.52 | node-to-node mesh | up | 16:41:20 | Established |
| 10.0.0.53 | node-to-node mesh | up | 16:41:20 | Established |
| 10.0.0.54 | node-to-node mesh | up | 16:41:20 | Established |
+--------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
```
## 更改ipip model从always到CrossSubnet。
```sh
# 当前通信我们是双层嵌套IP，即ipip模式，会自动生成一个tune0网卡，所有的流量都从这个网卡通信
[22:48:57 root@master1 cmd]#ifconfig
tunl0: flags=193<UP,RUNNING,NOARP> mtu 1480
inet 10.244.0.1 netmask 255.255.255.255
tunnel txqueuelen 1000 (IPIP Tunnel)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[22:54:58 root@master1 cmd]#ip route list
default via 10.0.0.2 dev eth0 proto static
10.0.0.0/24 dev eth0 proto kernel scope link src 10.0.0.50
blackhole 10.244.0.0/24 proto bird
10.244.0.5 dev calif478a5ea502 scope link
10.244.0.6 dev cali260b8463b8f scope link
10.244.0.7 dev cali21f5341d0ac scope link
10.244.3.0/24 via 10.0.0.53 dev tunl0 proto bird onlink
10.244.4.0/24 via 10.0.0.54 dev tunl0 proto bird onlink
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
[22:45:00 root@master1 cmd]#kubectl calico get ipPools default-ipv4-ippool -o yaml
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
creationTimestamp: "2021-12-11T15:49:12Z"
name: default-ipv4-ippool
resourceVersion: "2012"
uid: 894ee240-f66d-47a0-96ca-b634d20f2803
spec:
blockSize: 24
cidr: 10.244.0.0/16
ipipMode: Always
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
# 更改网卡模式到CrossSubnet，即通过实际的物理网卡eth0通信
[22:51:05 root@master1 cmd]#cat default-ipv4-ippool.yaml
apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
name: default-ipv4-ippool
spec:
blockSize: 24
cidr: 10.244.0.0/16
ipipMode: CrossSubnet
natOutgoing: true
nodeSelector: all()
vxlanMode: Never
[22:55:39 root@master1 cmd]#kubectl calico apply -f default-ipv4-ippool.yaml
Successfully applied 1 'IPPool' resource(s)
[22:55:41 root@master1 cmd]#ip route list
default via 10.0.0.2 dev eth0 proto static
10.0.0.0/24 dev eth0 proto kernel scope link src 10.0.0.50
blackhole 10.244.0.0/24 proto bird
10.244.0.5 dev calif478a5ea502 scope link
10.244.0.6 dev cali260b8463b8f scope link
10.244.0.7 dev cali21f5341d0ac scope link
10.244.3.0/24 via 10.0.0.53 dev eth0 proto bird
10.244.4.0/24 via 10.0.0.54 dev eth0 proto bird
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
```
## 反射器角色 - 从BGP peer到BGP reflecter
### 0 查看现在的node网络状态，是点对点模式
```sh
[00:44:06 root@master1 cmd]#kubectl calico node status
Calico process is running.
IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+--------------+-------------------+-------+----------+-------------+
| 10.0.0.51 | node-to-node mesh | up | 16:41:21 | Established |
| 10.0.0.52 | node-to-node mesh | up | 16:41:20 | Established |
| 10.0.0.53 | node-to-node mesh | up | 16:41:20 | Established |
| 10.0.0.54 | node-to-node mesh | up | 16:41:20 | Established |
+--------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
```
### 1 定义反射器角色
```sh
[23:10:14 root@master1 calico]#cat 01-calico-reflector-master.yaml
apiVersion: projectcalico.org/v3
kind: Node
metadata:
labels:
route-reflector: true
name: master1
spec:
bgp:
ipv4Address: 10.0.0.50/16
ipv4IPIPTunnelAddr: 10.244.0.1
routeReflectorClusterID: 1.1.1.1
[23:10:55 root@master1 calico]#kubectl calico apply -f 01-calico-reflector-master.yaml
Successfully applied 1 'Node' resource(s)
```
### 2 后端节点使用反射器
```sh
# 更改到BGP reflecter网络模型
[23:11:04 root@master1 calico]#cat 02-calico-reflector-bgppeer.yaml
kind: BGPPeer
apiVersion: projectcalico.org/v3
metadata:
name: bgppeer-demo
spec:
nodeSelector: all()
peerSelector: route-reflector=="true"
[23:12:09 root@master1 calico]#kubectl calico apply -f 02-calico-reflector-bgppeer.yaml
Successfully applied 1 'BGPPeer' resource(s)
# 查看node状态，同时存在点对点和反射网络模型
[00:44:58 root@master1 calico]#kubectl calico node status
Calico process is running.
IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+--------------+-------------------+-------+----------+-------------+
| 10.0.0.51 | node-to-node mesh | up | 16:41:22 | Established |
| 10.0.0.52 | node-to-node mesh | up | 16:41:21 | Established |
| 10.0.0.53 | node-to-node mesh | up | 16:41:21 | Established |
| 10.0.0.54 | node-to-node mesh | up | 16:41:21 | Established |
| 10.0.0.51 | node specific | start | 16:44:58 | Idle |
| 10.0.0.52 | node specific | start | 16:44:58 | Idle |
| 10.0.0.53 | node specific | start | 16:44:58 | Idle |
| 10.0.0.54 | node specific | start | 16:44:58 | Idle |
+--------------+-------------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
```
### 3 关闭默认的网格效果
```sh
[23:13:19 root@master1 calico]#cat 03-calico-reflector-defaultconfig.yaml
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
logSeverityScreen: Info
nodeToNodeMeshEnabled: false
asNumber: 63400
[23:13:50 root@master1 calico]#kubectl calico apply -f 03-calico-reflector-defaultconfig.yaml
Successfully applied 1 'BGPConfiguration' resource(s)
# 查看node状态，检查是否删除了点对点网络
[00:45:25 root@master1 calico]#kubectl calico node status
Calico process is running.
IPv4 BGP status
+--------------+---------------+-------+----------+-------------+
| PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO |
+--------------+---------------+-------+----------+-------------+
| 10.0.0.51 | node specific | up | 16:45:26 | Established |
| 10.0.0.52 | node specific | up | 16:45:25 | Established |
| 10.0.0.53 | node specific | up | 16:45:26 | Established |
| 10.0.0.54 | node specific | up | 16:45:26 | Established |
+--------------+---------------+-------+----------+-------------+
IPv6 BGP status
No IPv6 peers found.
```
## 网络策略
### 准备环境
```sh
[22:06:09 root@master1 calico]#cat 04-calico-deployment-test.yaml
apiVersion: v1
kind: Namespace
metadata:
name: develop
labels:
kubernetes.io/metadata.name: develop
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: pod-deployment1
name: pod-deployment1
namespace: develop
spec:
replicas: 3
selector:
matchLabels:
app: pod-deployment1
template:
metadata:
labels:
app: pod-deployment1
spec:
containers:
- image: 10.0.0.55:80/mykubernetes/pod_test:v0.1
name: pod-test
---
apiVersion: v1
kind: Service
metadata:
labels:
app: pod-deployment1
name: pod-deployment1
namespace: develop
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: pod-deployment1
type: ClusterIP
[22:12:35 root@master1 calico]#kubectl get pods -n develop -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod-deployment1-85bd7684f6-grsw9 1/1 Running 0 32s 10.244.3.4 node1.noisedu.cn <none> <none>
pod-deployment1-85bd7684f6-njn6x 1/1 Running 0 32s 10.244.3.5 node1.noisedu.cn <none> <none>
pod-deployment1-85bd7684f6-p9plh 1/1 Running 0 32s 10.244.4.5 node2.noisedu.cn <none> <none>
# 皆能正常访问
[22:12:47 root@master1 calico]#curl 10.244.3.4
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
[22:13:01 root@master1 calico]#curl 10.244.3.5
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-njn6x, ServerIP: 10.244.3.5!
[22:13:09 root@master1 calico]#curl 10.244.4.5
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-p9plh, ServerIP: 10.244.4.5!
```
### 默认策略
```sh
# 采用默认规则，虽然policy type定义了ingress和egress，但是未定义任何的ingress字段
[22:13:12 root@master1 calico]#cat 05-calico-networkpolicy-denyall.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: deny-all-ingress
namespace: develop
spec:
podSelector: {}
policyTypes: ["Ingress", "Egress"]
[22:20:08 root@master1 calico]#kubectl apply -f 05-calico-networkpolicy-denyall.yaml
networkpolicy.networking.k8s.io/deny-all-ingress created
# 测试，皆ping不通
[22:20:20 root@master1 calico]#curl 10.244.3.4
^C
[22:21:51 root@master1 calico]#curl 10.244.3.5
^C
[22:22:01 root@master1 calico]#curl 10.244.4.5
^C
# 设置默认的允许pod管控策略，ingress和egress未写字段，默认允许所有字段
[22:23:01 root@master1 calico]#cat 06-calico-networkpolicy-denyall.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: deny-all-ingress
namespace: develop
spec:
podSelector: {}
policyTypes: ["Ingress", "Egress"]
egress:
- {}
ingress:
- {}
[22:23:06 root@master1 calico]#kubectl apply -f 06-calico-networkpolicy-denyall.yaml
networkpolicy.networking.k8s.io/deny-all-ingress created
# 测试
[22:23:20 root@master1 calico]#curl 10.244.3.4
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
[22:23:28 root@master1 calico]#curl 10.244.3.5
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-njn6x, ServerIP: 10.244.3.5!
[22:23:30 root@master1 calico]#curl 10.244.4.5
kubernetes pod-test v0.1!! ClientIP: 10.0.0.50, ServerName: pod-deployment1-85bd7684f6-p9plh, ServerIP: 10.244.4.5!
# 仅当前命名空间的所有节点可访问
[22:27:31 root@master1 calico]#cat 07-calico-networkpolicy-denyall.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: deny-all-ingress
namespace: develop
spec:
podSelector: {}
policyTypes: ["Ingress", "Egress"]
egress:
- to:
- podSelector: {}
ingress:
- from:
- podSelector: {}
[22:27:33 root@master1 calico]#kubectl apply -f 07-calico-networkpolicy-denyall.yaml
networkpolicy.networking.k8s.io/deny-all-ingress created
[22:28:03 root@master1 calico]#curl 10.244.3.4
^C
[22:28:37 root@master1 calico]#kubectl get svc -n develop
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
pod-deployment1 ClusterIP 10.101.254.69 <none> 80/TCP 16m
# 测试，在主机ping不通
[22:29:00 root@master1 calico]#curl 10.101.254.69
^C
# 在一个同命名空间可通
[22:29:28 root@master1 calico]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.3.6, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
```
### 同ns策略
```sh
# 创建两个测试pod
[22:39:10 root@master1 ~]#cd ~
[22:39:10 root@master1 ~]#kubectl run pod-test1 --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
#
[22:39:12 root@master1 ~]#cd ~
[22:39:12 root@master1 ~]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
#
[22:40:04 root@master1 calico]#kubectl get pod -n develop -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod-test 1/1 Running 0 26s 10.244.4.6 node2.noisedu.cn <none> <none>
pod-test1 1/1 Running 0 21s 10.244.3.7 node1.noisedu.cn <none> <none>
# 应用策略，仅当前命名空间的所有节点可访问，并且拒绝10.244.4.6的访问
[22:41:56 root@master1 calico]#cat 08-calico-networkpolicy-denyall.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: deny-all-ingress
namespace: develop
spec:
podSelector: {}
policyTypes: ["Ingress", "Egress"]
egress:
- to:
- podSelector: {}
ingress:
- from:
- ipBlock:
cidr: 10.244.0.0/16
except:
- 10.244.4.6/32
- podSelector:
matchLabels:
run: pod-test1
ports:
- protocol: TCP
port: 80
# 测试应用前
[22:39:12 root@master1 ~]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.4.6, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
#
[22:39:10 root@master1 ~]#kubectl run pod-test1 --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.3.7, ServerName: pod-deployment1-85bd7684f6-njn6x, ServerIP: 10.244.3.5!
#
# 应用策略
[22:41:59 root@master1 calico]#kubectl apply -f 08-calico-networkpolicy-denyall.yaml
networkpolicy.networking.k8s.io/deny-all-ingress created
# 测试应用后，pod-test1成功，pod-test失败
[22:39:10 root@master1 ~]#kubectl run pod-test1 --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.3.7, ServerName: pod-deployment1-85bd7684f6-njn6x, ServerIP: 10.244.3.5!
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.3.7, ServerName: pod-deployment1-85bd7684f6-p9plh, ServerIP: 10.244.4.5!
#
[22:39:12 root@master1 ~]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.4.6, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
# curl 10.101.254.69
# 增加出栈规则，允许udp 53端口和tcp80端口
[22:45:14 root@master1 calico]#cat 09-calico-networkpolicy-egress.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: egress-controller
namespace: develop
spec:
podSelector:
matchLabels:
run: pod-test1
policyTypes: ["Egress"]
egress:
- to:
ports:
- protocol: UDP
port: 53
- to:
- podSelector:
matchLabels:
run: pod-test1
ports:
- protocol: TCP
port: 80
```
### 跨ns策略
```sh
# 允许不是default的ns访问，其他拒绝
[22:46:50 root@master1 calico]#cat 10-calico-networkpolicy-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: ingress-controller
namespace: develop
spec:
podSelector: {}
policyTypes: ["Ingress"]
ingress:
- from:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values: [develop, kube-system, logs]
- from:
- namespaceSelector:
matchExpressions:
- {key: kubernetes.io/metadata.name, operator: NotIn, values: ["default"]}
[22:46:53 root@master1 calico]#kubectl apply -f 10-calico-networkpolicy-ingress.yaml
networkpolicy.networking.k8s.io/ingress-controller created
# 测试
[22:49:01 root@master1 ~]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
kubernetes pod-test v0.1!! ClientIP: 10.244.4.7, ServerName: pod-deployment1-85bd7684f6-grsw9, ServerIP: 10.244.3.4!
#
[22:49:07 root@master1 ~]#kubectl run pod-test1 --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
^C
#
# networkpolcy
```
### NetworkPolicy
```sh
[22:51:22 root@master1 calico]#cat 11-calico-networkpolicy-global.yaml
apiVersion: crd.projectcalico.org/v1
kind: GlobalNetworkPolicy
metadata:
name: namespaces-default
spec:
order: 0.0
namespaceSelector: name not in {"kube-system","kubernetes-dashboard","logs","monitoring"}
types: ["Ingress", "Egress"]
ingress:
- action: Allow
source:
namespaceSelector: name in {"kube-system","kubernetes-dashboard","logs","monitoring"}
egress:
- action: Allow
[22:51:24 root@master1 calico]#kubectl apply -f 11-calico-networkpolicy-global.yaml
globalnetworkpolicy.crd.projectcalico.org/namespaces-default created
[22:52:00 root@master1 calico]#kubectl get globalnetworkpolicies.crd.projectcalico.org
NAME AGE
namespaces-default 8s
[22:52:08 root@master1 calico]#kubectl get globalnetworkpolicy
NAME AGE
namespaces-default 17s
[22:52:17 root@master1 calico]#kubectl describe globalnetworkpolicies.crd.projectcalico.org namespaces-default
Name: namespaces-default
Namespace:
Labels: <none>
Annotations: <none>
API Version: crd.projectcalico.org/v1
Kind: GlobalNetworkPolicy
Metadata:
Creation Timestamp: 2022-01-02T14:52:00Z
Generation: 1
Managed Fields:
API Version: crd.projectcalico.org/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:kubectl.kubernetes.io/last-applied-configuration:
f:spec:
.:
f:egress:
f:ingress:
f:namespaceSelector:
f:order:
f:types:
Manager: kubectl-client-side-apply
Operation: Update
Time: 2022-01-02T14:52:00Z
Resource Version: 126481
UID: 31210e0b-b665-4a82-bafb-c400d3d0b248
Spec:
Egress:
Action: Allow
Ingress:
Action: Allow
Source:
Namespace Selector: name in {"kube-system","kubernetes-dashboard","logs","monitoring"}
Namespace Selector: name not in {"kube-system","kubernetes-dashboard","logs","monitoring"}
Order: 0
Types:
Ingress
Egress
Events: <none>
# 测试，都是超时，因为是defaul的ns，这个global设置不允许default的pod访问。
[22:49:07 root@master1 ~]#kubectl run pod-test1 --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
^C
[22:49:01 root@master1 ~]#kubectl run pod-test --image=10.0.0.55:80/mykubernetes/nginx:1.21.3 -n develop --rm -it --command -- sh
If you don't see a command prompt, try pressing enter.
# curl 10.101.254.69
^C
#
```
浙公网安备 33010602011771号