# kubectl get daemonset calico-node -n kube-system -o jsonpath='{.spec.template.spec.containers[0].image}' ; echo
calico/node:v3.23.2



# kubectl describe daemonset calico-node -n kube-system | grep Image
    Image:      calico/cni:v3.23.2
    Image:      calico/cni:v3.23.2
    Image:      calico/node:v3.23.2

# kubectl get pod -n kube-system | grep calico-node
calico-node-4279g                             1/1     Running   1 (39d ago)      68d
calico-node-59fr6                             1/1     Running   3 (73d ago)      75d
calico-node-8w7vl                             1/1     Running   1 (38d ago)      68d
calico-node-jjgpm                             1/1     Running   2 (73d ago)      75d
calico-node-jvw65                             1/1     Running   0                75d
calico-node-lrdhb                             1/1     Running   1 (38d ago)      68d
calico-node-plnjq                             1/1     Running   1 (39d ago)      68d
calico-node-qfqrn                             1/1     Running   3 (67d ago)      75d
calico-node-rngxp                             1/1     Running   3 (73d ago)      75d
calico-node-twzqr                             1/1     Running   1 (39d ago)      68d
calico-node-vshqb                             1/1     Running   1 (39d ago)      68d
calico-node-xt5kz                             1/1     Running   1 (38d ago)      68d
calico-node-z5gjd                             1/1     Running   0                75d

# kubectl get nodes -o wide
NAME           STATUS                     ROLES                  AGE    VERSION    INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
k8s-master1    Ready                      control-plane,master   140d   v1.22.12   172.16.0.120   <none>        Ubuntu 18.04.4 LTS   4.15.0-213-generic   docker://20.10.8
k8s-master2    Ready                      control-plane,master   140d   v1.22.12   172.16.0.121   <none>        Ubuntu 18.04.4 LTS   4.15.0-213-generic   docker://20.10.8
k8s-master3    Ready                      control-plane,master   140d   v1.22.12   172.16.0.122   <none>        Ubuntu 18.04.4 LTS   4.15.0-213-generic   docker://20.10.8
k8s-worker-1   Ready                      worker                 29d    v1.22.12   172.16.0.130   <none>        Ubuntu 20.04.6 LTS   5.4.0-155-generic    docker://20.10.23
k8s-worker-2   Ready                      worker                 29d    v1.22.12   172.16.0.131   <none>        Ubuntu 20.04.6 LTS   5.4.0-155-generic    docker://20.10.23
k8s-worker-3   Ready                      worker                 29d    v1.22.12   172.16.0.132   <none>        Ubuntu 20.04.6 LTS   5.4.0-155-generic    docker://20.10.23
k8s-worker-4   Ready,SchedulingDisabled   worker                 18d    v1.22.12   172.16.0.133   <none>        Ubuntu 20.04.6 LTS   5.15.0-78-generic    docker://20.10.8



# kubectl describe ds -n kube-system calico-node | grep Image
    Image:      calico/cni:v3.23.2
    Image:      calico/cni:v3.23.2
    Image:      calico/node:v3.23.2

root@k8s-worker-1:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         172.16.0.1      0.0.0.0         UG    0      0        0 ens160
10.233.68.0     172.16.0.120    255.255.255.0   UG    0      0        0 tunl0
10.233.72.0     172.16.0.133    255.255.255.0   UG    0      0        0 tunl0
10.233.74.0     172.16.0.131    255.255.255.0   UG    0      0        0 tunl0
10.233.83.0     172.16.0.122    255.255.255.0   UG    0      0        0 tunl0
10.233.87.0     172.16.0.132    255.255.255.0   UG    0      0        0 tunl0
10.233.88.0     172.16.0.121    255.255.255.0   UG    0      0        0 tunl0
10.233.89.0     0.0.0.0         255.255.255.0   U     0      0        0 *
10.233.89.217   0.0.0.0         255.255.255.255 UH    0      0        0 caliddde6aa41aa
10.233.89.218   0.0.0.0         255.255.255.255 UH    0      0        0 cali8ee0b615a05
10.233.89.219   0.0.0.0         255.255.255.255 UH    0      0        0 calia8562f237aa
10.233.89.220   0.0.0.0         255.255.255.255 UH    0      0        0 calib780a7d3bc9
10.233.89.221   0.0.0.0         255.255.255.255 UH    0      0        0 caliadfd805b63c
10.233.89.222   0.0.0.0         255.255.255.255 UH    0      0        0 cali52be4ea9497
10.233.89.223   0.0.0.0         255.255.255.255 UH    0      0        0 cali54e86f2c93e
10.233.89.224   0.0.0.0         255.255.255.255 UH    0      0        0 cali6796fb4c6a8
10.233.89.225   0.0.0.0         255.255.255.255 UH    0      0        0 calib1621a7eac7
10.233.89.226   0.0.0.0         255.255.255.255 UH    0      0        0 cali4d239ec8772
10.233.89.227   0.0.0.0         255.255.255.255 UH    0      0        0 calid9819a15597
10.233.89.228   0.0.0.0         255.255.255.255 UH    0      0        0 cali0b5fd109ea1
10.233.89.229   0.0.0.0         255.255.255.255 UH    0      0        0 cali09304935b77
10.233.89.231   0.0.0.0         255.255.255.255 UH    0      0        0 caliea2cfd262d1
10.233.89.232   0.0.0.0         255.255.255.255 UH    0      0        0 cali303ff4ac14e
10.233.89.233   0.0.0.0         255.255.255.255 UH    0      0        0 calia3f8635265e
172.16.0.0      0.0.0.0         255.255.192.0   U     0      0        0 ens160
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0


root@k8s-worker-1:~# ip route 
default via 172.16.0.1 dev ens160 proto static 
10.233.68.0/24 via 172.16.0.120 dev tunl0 proto bird onlink 
10.233.72.0/24 via 172.16.0.133 dev tunl0 proto bird onlink 
10.233.74.0/24 via 172.16.0.131 dev tunl0 proto bird onlink 
10.233.83.0/24 via 172.16.0.122 dev tunl0 proto bird onlink 
10.233.87.0/24 via 172.16.0.132 dev tunl0 proto bird onlink 
10.233.88.0/24 via 172.16.0.121 dev tunl0 proto bird onlink 
blackhole 10.233.89.0/24 proto bird 
10.233.89.217 dev caliddde6aa41aa scope link 
10.233.89.218 dev cali8ee0b615a05 scope link 
10.233.89.219 dev calia8562f237aa scope link 
10.233.89.220 dev calib780a7d3bc9 scope link 
10.233.89.221 dev caliadfd805b63c scope link 
10.233.89.222 dev cali52be4ea9497 scope link 
10.233.89.223 dev cali54e86f2c93e scope link 
10.233.89.224 dev cali6796fb4c6a8 scope link 
10.233.89.225 dev calib1621a7eac7 scope link 
10.233.89.226 dev cali4d239ec8772 scope link 
10.233.89.227 dev calid9819a15597 scope link 
10.233.89.228 dev cali0b5fd109ea1 scope link 
10.233.89.229 dev cali09304935b77 scope link 
10.233.89.231 dev caliea2cfd262d1 scope link 
10.233.89.232 dev cali303ff4ac14e scope link 
10.233.89.233 dev calia3f8635265e scope link 
172.16.0.0/18 dev ens160 proto kernel scope link src 172.16.0.130 
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 

root@k8s-worker-1:~# ip addr show ens160 
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:0c:29:8f:3e:70 brd ff:ff:ff:ff:ff:ff
    inet 172.16.0.130/18 brd 172.16.63.255 scope global ens160
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe8f:3e70/64 scope link 
       valid_lft forever preferred_lft forever


root@k8s-worker-1:~# ip addr show tunl0 
26: tunl0@NONE: <NOARP,UP,LOWER_UP> mtu 1480 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
    inet 10.233.89.0/32 scope global tunl0
       valid_lft forever preferred_lft forever


root@k8s-worker-1:~# ip tunnel 
tunl0: any/ip remote any local any ttl inherit nopmtudisc

root@k8s-master1:~# kubectl get ds -n kube-system calico-node -o yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: calico-node
  namespace: kube-system
...
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: calico-node
  template:
    metadata:
      creationTimestamp: null
      labels:
        k8s-app: calico-node
    spec:
      containers:
      - env:
...
        - name: CALICO_IPV4POOL_IPIP # 在IPv4类型的地址池上启用的IPIP及其类型，支持3种可用值
          value: Always # Always（全局流量）、Cross-SubNet（跨子网流量）和 Never 3种可用值
        - name: CALICO_IPV4POOL_VXLAN # 是否在IPV4地址池上启用VXLAN隧道协议，取值及意义与Flannel的VXLAN后端相同； 
          value: Never # 但在全局流量启用VXLAN时将完全不再需要BGP网络，建议将相关的组件禁用
...
        image: calico/node:v3.23.2

# IPV4地址池的定义，其值需要与 kube-controller-manager 的 “--cluster-network” 选项的值保持一致，以下环境变量默认处于注释状态
- name: CALICO_IPV4POOL_CIDR
  value: "192.168.0.0/16"

# Calico默认以26位子网掩码切分地址池并将各子网配置给集群中的节点，若需要使用其他的掩码长度，则需要定义如下环境变量
- name: CALICO_IPV4POOL_BLOCK_SIZE
  value: "24"

# Calico 默认并不会从 Node.Spec.PodCIDR 中分配地址，但可通过将如下变量设置为 “true” 并结合 host-local 这一 IPAM 插件以强制从 PodCIDR 中分配地址
- name: USE_POD_CIDR
  value: "false"

"ipam": {
    "type": "host-local",
    "subnet": "usePodCidr"
},

apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
  datastoreType: "kubernetes"
  kubeconfig: "/etc/kubernetes/admin.conf"

curl -L https://github.com/projectcalico/calico/releases/download/v3.23.5/calicoctl-linux-amd64 -o calicoctl

# kubectl get ippool default-ipv4-ippool -o yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  name: default-ipv4-ippool
spec:
  allowedUses:
  - Workload
  - Tunnel
  blockSize: 24
  cidr: 10.233.64.0/18
  ipipMode: Always
  natOutgoing: true
  nodeSelector: all()
  vxlanMode: Never

allowedUses:
  - Workload
  - Tunnel
  - Gateway

nodeSelector:
  key1: value1
  key2: value2

# kubectl get ippool
NAME                  AGE
default-ipv4-ippool   72d


# kubectl describe ippool default-ipv4-ippool
...
  Cidr:           10.233.64.0/18
...

# kubectl get ippool
NAME                  AGE
default-ipv4-ippool   72d


# kubectl describe ippool default-ipv4-ippool
...
  Cidr:           10.233.64.0/18
...

# vim my.ippool-1.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  name: my.ippool-1
spec:
  cidr: 10.200.64.0/24
  ipipMode: CrossSubnet
  natOutgoing: true
  disabled: false
  nodeSelector: all()
  allowedUses:
  - Workload
  - Tunnel 

# kubectl get ippool
NAME                  AGE
default-ipv4-ippool   72d
my.ippool-1           31s


# kubectl describe ippool my.ippool-1 
...
  Cidr:           10.200.64.0/24
...

# vim demoapp.yaml
apiVersion: v1
kind: Pod
metadata:
  name: demoapp
  namespace: test
  annotations: # 注解
    cni.projectcalico.org/ipAddrs: "[\"10.233.65.177\"]" # 明确指定分配的IP
spec:
  containers:
  - name: demoapp # 容器一
    image: ikubernetes/demoapp:v1.0
    imagePullPolicy: IfNotPresent
  - name: curl # 容器二
    image: curlimages/curl:8.1.0
    command:
      - tail
    args:
      - "-f"
      - /etc/hosts
    imagePullPolicy: IfNotPresent

# kubectl describe pod -n test demoapp
Name:         demoapp
Namespace:    test
Priority:     0
Node:         k8s-worker2/172.16.0.124
Start Time:   Thu, 01 Jun 2023 15:27:30 +0800
Labels:       <none>
Annotations:  cni.projectcalico.org/containerID: 6917fee7bbdd6400defaeda7f0f9f6545e80f2aff339e4ca868dbe812b9e929f
              cni.projectcalico.org/ipAddrs: ["10.233.65.177"]
              cni.projectcalico.org/podIP: 10.233.65.177/32
              cni.projectcalico.org/podIPs: 10.233.65.177/32
Status:       Running
IP:           10.233.65.177
IPs:
  IP:  10.233.65.177
...

# vim demoapp_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp
  namespace: test
spec:
  replicas: 3 # 创建3个Pod副本
  selector:
    matchLabels:
      app: demoapp
  template:
    metadata:
      name: demoapp
      labels:
        app: demoapp
      annotations: # 注解
        cni.projectcalico.org/ipAddrs: "[\"10.233.65.177\", \"10.233.65.178\"]" # 分配多个IP
    spec:
      containers:
      - name: demoapp
        image: 172.16.0.120:30002/example-middleware/logger-processor@sha256:1c973d59cec3915f8c50c86f102a4d9890855a4f576bcac5bec716e6b5597839
        env:
        - name: NTP_SERVER1
          value: "172.16.0.125"
        - name: NTP_SERVER2
          value: "172.16.0.127"
        securityContext:
          capabilities:
            add: ['CAP_SYS_TIME']



# 所有Pod都将无法分配IP
# kubectl get pod -n test -o wide
NAME                       READY   STATUS              RESTARTS   AGE     IP       NODE          NOMINATED NODE   READINESS GATES
demoapp-67578686fb-8kv7p   0/1     ContainerCreating   0          2m37s   <none>   k8s-worker3   <none>           <none>
demoapp-67578686fb-bvtlg   0/1     ContainerCreating   0          2m37s   <none>   k8s-worker1   <none>           <none>
demoapp-67578686fb-swk5j   0/1     ContainerCreating   0          2m37s   <none>   k8s-worker2   <none>           <none>



# 报错
# kubectl describe pod -n test demoapp-67578686fb-bvtlg 
...
  Warning  FailedCreatePodSandBox  2m41s                   kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "572a7a238eaefae2da6a2ce5e2ae6834187aa7f450365c5e9344815e56545a50" network for pod "demoapp-67578686fb-bvtlg": networkPlugin cni failed to set up pod "demoapp-67578686fb-bvtlg_test" network: cannot have more than one IPv4 address for "cni.projectcalico.org/ipAddrs" annotation
  Normal   SandboxChanged          2m33s (x12 over 2m58s)  kubelet            Pod sandbox changed, it will be killed and re-created.
  Warning  FailedCreatePodSandBox  2m32s (x4 over 2m39s)   kubelet            (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to set up sandbox container "1e05db9586184396db5add5e44bd1fd73a5f83a07b976165a997bacdbc394a33" network for pod "demoapp-67578686fb-bvtlg": networkPlugin cni failed to set up pod "demoapp-67578686fb-bvtlg_test" network: cannot have more than one IPv4 address for "cni.projectcalico.org/ipAddrs" annotation

# kubectl get ippool
NAME                  AGE
default-ipv4-ippool   72d
my.ippool-1           47m


# kubectl get ippool default-ipv4-ippool -o jsonpath='{.spec.cidr}' ; echo
10.233.64.0/18


# kubectl get ippool my.ippool-1 -o jsonpath='{.spec.cidr}' ; echo
10.200.64.0/24

# vim demoapp.yaml
apiVersion: v1
kind: Pod
metadata:
  name: demoapp
  namespace: test
  annotations: # 注解
    cni.projectcalico.org/ipv4pools: '["my.ippool-1"]' # 使用该ippool中的ip
spec:
  containers:
  - name: demoapp # 容器一
    image: ikubernetes/demoapp:v1.0
    imagePullPolicy: IfNotPresent
  - name: curl # 容器二
    image: curlimages/curl:8.1.0
    command:
      - tail
    args:
      - "-f"
      - /etc/hosts
    imagePullPolicy: IfNotPresent

# kubectl describe pod -n test demoapp
Name:         demoapp
Namespace:    test
Priority:     0
Node:         k8s-worker2/172.16.0.124
Start Time:   Thu, 01 Jun 2023 16:47:50 +0800
Labels:       <none>
Annotations:  cni.projectcalico.org/containerID: 2ba2dceb3057fd11e3523c8dc3cf14b115745b9e91b5874193fe9cf19d606ab6
              cni.projectcalico.org/ipv4pools: ["my.ippool-1"]
              cni.projectcalico.org/podIP: 10.200.64.0/32
              cni.projectcalico.org/podIPs: 10.200.64.0/32
Status:       Running
IP:           10.200.64.0 # 分配的地址位于 my.ippool-1
IPs:
  IP:  10.200.64.0
...

# vim demoapp_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp
  namespace: test
spec:
  replicas: 3 # 创建3个Pod副本
  selector:
    matchLabels:
      app: demoapp
  template:
    metadata:
      name: demoapp
      labels:
        app: demoapp
      annotations: # 注解
        cni.projectcalico.org/ipv4pools: '["my.ippool-1"]' # 指定地址池
    spec:
      containers:
      - name: demoapp
        image: 172.16.0.120:30002/example-middleware/logger-processor@sha256:1c973d59cec3915f8c50c86f102a4d9890855a4f576bcac5bec716e6b5597839
        env:
        - name: NTP_SERVER1
          value: "172.16.0.125"
        - name: NTP_SERVER2
          value: "172.16.0.127"
        securityContext:
          capabilities:
            add: ['CAP_SYS_TIME']



# ok
# kubectl get pod -n test -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP              NODE          NOMINATED NODE   READINESS GATES
demoapp-5d7978f7f9-4kczb   1/1     Running   0          28s   10.200.64.1     k8s-worker2   <none>           <none>
demoapp-5d7978f7f9-bzrtz   1/1     Running   0          28s   10.200.64.64    k8s-worker1   <none>           <none>
demoapp-5d7978f7f9-c8jp9   1/1     Running   0          28s   10.200.64.192   k8s-worker3   <none>           <none>

# vim america_ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  name: america-ippool
spec:
  cidr: 10.200.66.0/24
  ipipMode: CrossSubnet
  natOutgoing: true
  disabled: false
  nodeSelector: all()
  allowedUses:
  - Workload
  - Tunnel 


# kubectl get ippool
NAME                  AGE
america-ippool        7s
default-ipv4-ippool   72d
my.ippool-1           167m



# kubectl describe ippool america-ippool
...
  Cidr:           10.200.66.0/24
...

# vim america-node.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: america-node
  annotations: # 注解
    cni.projectcalico.org/ipv4pools: '["america-ippool"]' # 指定地址池



# kubectl apply -f america-node.yaml 
namespace/america-node created


# kubectl describe ns america-node 
Name:         america-node
Labels:       kubernetes.io/metadata.name=america-node
              kubesphere.io/namespace=america-node
Annotations:  cni.projectcalico.org/ipv4pools: ["america-ippool"]
Status:       Active
...

# vim demoapp_deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp
  namespace: america-node # 指定namespace
spec:
  replicas: 3 # 创建3个Pod副本
  selector:
    matchLabels:
      app: demoapp
  template:
    metadata:
      name: demoapp
      labels:
        app: demoapp
    spec:
      containers:
      - name: demoapp
        image: 172.16.0.120:30002/example-middleware/logger-processor@sha256:1c973d59cec3915f8c50c86f102a4d9890855a4f576bcac5bec716e6b5597839
        env:
        - name: NTP_SERVER1
          value: "172.16.0.125"
        - name: NTP_SERVER2
          value: "172.16.0.127"
        securityContext:
          capabilities:
            add: ['CAP_SYS_TIME']



# kubectl get pod -n america-node -o wide
NAME                       READY   STATUS    RESTARTS   AGE   IP              NODE          NOMINATED NODE   READINESS GATES
demoapp-759f97d8cb-24mx2   1/1     Running   0          16s   10.200.66.192   k8s-worker3   <none>           <none>
demoapp-759f97d8cb-47g8x   1/1     Running   0          16s   10.200.66.0     k8s-worker2   <none>           <none>
demoapp-759f97d8cb-qxvns   1/1     Running   0          16s   10.200.66.64    k8s-worker1   <none>           <none>

# vim america_ippool.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  name: america-ippool
spec:
  cidr: 10.200.66.0/24
  ipipMode: CrossSubnet
  natOutgoing: true
  disabled: false
  nodeSelector: all()
  allowedUses:
  - Workload
  - Tunnel 

# vim my.ippool-1.yaml
apiVersion: crd.projectcalico.org/v1
kind: IPPool
metadata:
  name: my.ippool-1
spec:
  cidr: 10.200.64.0/24
  ipipMode: CrossSubnet
  natOutgoing: true
  disabled: false
  nodeSelector: all()
  allowedUses:
  - Workload
  - Tunnel

apiVersion: v1
kind: Namespace
metadata:
  name: america-node
  annotations: # 注解
    cni.projectcalico.org/ipv4pools: '["america-ippool"]' # 指定地址池

apiVersion: v1
kind: Namespace
metadata:
  name: test 
  annotations: # 注解
    cni.projectcalico.org/ipv4pools: '["my.ippool-1"]' # 指定地址池

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp-america
  namespace: america-node # 指定namespace
spec:
  replicas: 3 # 创建3个Pod副本
  selector:
    matchLabels:
      app: demoapp-america
  template:
    metadata:
      name: demoapp-america
      labels:
        app: demoapp-america
    spec:
      containers:
      - name: demoapp-america
        image: 172.16.0.120:30002/example-middleware/logger-processor@sha256:1c973d59cec3915f8c50c86f102a4d9890855a4f576bcac5bec716e6b5597839
        env:
        - name: NTP_SERVER1
          value: "172.16.0.125"
        - name: NTP_SERVER2
          value: "172.16.0.127"
        securityContext:
          capabilities:
            add: ['CAP_SYS_TIME']

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp-test
  namespace: test # 指定namespace
spec:
  replicas: 3 # 创建3个Pod副本
  selector:
    matchLabels:
      app: demoapp-test
  template:
    metadata:
      name: demoapp-test
      labels:
        app: demoapp-test
    spec:
      containers:
      - name: demoapp-test
        image: 172.16.0.120:30002/example-middleware/logger-processor@sha256:1c973d59cec3915f8c50c86f102a4d9890855a4f576bcac5bec716e6b5597839
        env:
        - name: NTP_SERVER1
          value: "172.16.0.125"
        - name: NTP_SERVER2
          value: "172.16.0.127"
        securityContext:
          capabilities:
            add: ['CAP_SYS_TIME']

root@demoapp-test-85fc4bd9f7-kkf8c:/var/lib/logger_processor# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: tunl0@NONE: <NOARP> mtu 1480 qdisc noop state DOWN group default qlen 1000
    link/ipip 0.0.0.0 brd 0.0.0.0
4: eth0@if191: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1480 qdisc noqueue state UP group default 
    link/ether 9e:7f:5a:bd:ea:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.200.64.192/32 scope global eth0
       valid_lft forever preferred_lft forever



root@demoapp-test-85fc4bd9f7-kkf8c:/var/lib/logger_processor# ping 10.200.66.21
PING 10.200.66.21 (10.200.66.21) 56(84) bytes of data.
64 bytes from 10.200.66.21: icmp_seq=1 ttl=62 time=0.206 ms
64 bytes from 10.200.66.21: icmp_seq=2 ttl=62 time=0.204 ms
64 bytes from 10.200.66.21: icmp_seq=3 ttl=62 time=0.213 ms

尽管功能上日渐丰富，但k8s自己的NetworkPolicy资源仍然具有相当的局限性，例如它没有明确的拒绝规则、缺乏对选择器高级表达式的支持、不支持应用层规则，以及没有集群范围的网络策略等。为了解决这些限制，Calico等提供了自有的策略CRD，包括NetworkPolicy和GlobalNetworkPolicy等，其中的NetworkPolicy CRD比Kubernetes NetworkPolicy API提供了更大的功能集，包括支持拒绝规则、规则解析级别以及应用层规则等，但相关的规则需要由calicoctl创建。


GlobalNetworkPolicy支持使用selector、serviceAccountSelector或namespaceSelector来选定网络策略的生效范围，默认为all()，即集群上的所有端点。下面的配置清单示例（globalnetworkpolicy-demo.yaml）为非系统类名称空间（本示例假设有kube-system、kubernetes-dashboard、logs和monitoring这4个）定义了一个通用的网络策略。

apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
  name: namespaces-default
spec:
  order: 0.0  # 策略叠加时的应用次序，数字越小越先应用，冲突时，后者会覆盖前者
  # 策略应用目标为非指定名称空间中的所有端点
namespaceSelector: name not in {"kube-system","kubernetes-dashboard","logs","monitoring"}
  types: ["Ingress", "Egress"]
  ingress:  # 入站流量规则
  - action: Allow  # 白名单
    source:
      # 策略生效目标中的端点可由下面系统名称空间中每个源端点访问任意端口
      namespaceSelector: name in {"kube-system","kubernetes-dashboard","logs","monitoring"}
  egress:  # 出站流量规则
  - action: Allow  # 允许所有





这些用于节点过滤的预选函数负责根据指定判定标准及各Node对象和当前Pod对象能否适配，它们按照用于实现的主要目标大体可分为如下几类。
	节点存储卷数量限制检测：MaxEBSVolumeCount、MaxGCEPDVolumeCount、 MaxCSIVolumeCount、MaxAzureDiskVolumeCount和MaxCinderVolumeCount。
	检测节点状态是否适合运行Pod：CheckNodeUnschedulable和CheckNodeLabelPresence。
	Pod与节点的匹配度检测：Hostname、PodFitsHostPorts、MatchNodeSelector、NoDiskConflict、PodFitsResources、PodToleratesNodeTaints、PodToleratesNodeNoExecuteTaints、CheckVolumeBinding和NoVolumeZoneConflict
	Pod间的亲和关系判定：MatchInterPodAffinity。
	将一组Pod打散至集群或特定的拓扑结构中：CheckServiceAffinity和EvenPodsSpread。
在Kubernetes Scheduler上启用相应的预选函数才能实现相关调度机制的节点过滤需求，下面给出了这些于Kubernetes v1.17版本中支持的各预选函数的简要功能，其中仅ServiceAffinity和CheckNodeLabelPresence支持自定义配置，余下的均为静态函数。
1）CheckNodeUnschedulable：检查节点是否被标识为Unschedulable，以及是否可将Pod调度于该类节点之上。
2）HostName：若Pod资源通过spec. nodeName明确指定了要绑定的目标节点，则节点名称与与该字段值相同的节点才会被保留。
3）PodFitsHostPorts：若Pod容器定义了ports.hostPort属性，该预选函数负责检查其值指定的端口是否已被节点上的其他容器或服务所占用，该端口已被占用的节点将被过滤掉。
4）MatchNodeSelector：若Pod资源规范上定义了spec.nodeSelector字段，则仅那些拥有匹配该标签选择器的标签的节点才会被保留。
5）NoDiskConflict：检查Pod对象请求的存储卷在此节点是否可用，不存在冲突则通过检查。
6）PodFitsResources：检查节点是否有足够资源（例如 CPU、内存和GPU等）满足Pod的运行需求；节点声明其资源可用容量，而Pod定义其资源需求（requests），于是调度器会判断节点是否有足够的可用资源运行Pod对象，无法满足则返回失败原因（例如，CPU或内存资源不足等）；调度器的评判资源消耗的标准是节点已分配资源量（各容器的requests值之和），而非其上的各Pod已用资源量，但那些在注解中标记为关键性（critical）的Pod资源则不受该预选函数控制。
7）PodToleratesNodeTaints：检查Pod的容忍度（spec.tolerations字段）是否能够容忍该节点上的污点（taints），不过，它仅关注具有NoSchedule和NoExecute两个效用标识的污点。
8）PodToleratesNodeNoExecuteTaints：检查Pod的容忍度是否能接纳节点上定义的NoExecute类型的污点。
9）CheckNodeLabelPresence：检查节点上某些标签的存在性，要检查的标签以及其可否存在则取决于用户的定义；在集群中的部署节点以regions/zones/racks类标签的拓扑方式编制，且基于该类标签对相应节点进行了位置标识时，预选函数可以根据位置标识将Pod调度至此类节点之上。
10）CheckServiceAffinity：根据调度的目标Pod对象所属的Service资源已关联的其他Pod对象的位置（所运行节点）来判断当前Pod可以运行的目标节点，其目的在于将同一Service对象的Pod放置在同一拓扑内（如同一个rack或zone）的节点上以提高效率。
11）MaxEBSVolumeCount：检查节点上已挂载的EBS存储卷数量是否超过了设置的最大值。
12）MaxGCEPDVolumeCount：检查节点上已挂载的GCE PD存储卷数量是否超过了设置的最大值，默认值为16。
13）MaxCSIVolumeCount：检查节点上已挂载的CSI存储卷数量是否超过了设置的最大值。
14）MaxAzureDiskVolumeCount：检查节点上已挂载的Azure Disk存储卷数量是否超过了设置的最大值，默认值为16。
15）MaxCinderVolumeCount：检查节点上已挂载的Cinder存储卷数量是否超过了设置的最大值。
16）CheckVolumeBinding：检查节点上已绑定和未绑定的PVC是否能满足Pod的存储卷需求，对于已绑定的PVC，此预选函数检查给定节点是否能兼容相应PV，而对于未绑定的PVC，预选函数搜索那些可满足PVC申请的可用PV，并确保它可与给定的节点兼容。
17）NoVolumeZoneConflict：在给定了存储故障域的前提下，检测节点上的存储卷是否可满足Pod定义的需求。
18）EvenPodsSpread：检查节点是否能满足Pod规范中topologySpreadConstraints字段中定义的约束以支持Pod的拓扑感知调度。
19）MatchInterPodAffinity：检查给定节点是否能满足Pod对象的亲和性或反亲和性条件，用于实现Pod亲和性调度或反亲和性调度。









	LeastRequestedPriority：优先将Pod打散至集群中的各节点之上，以试图让各节点有着近似的计算资源消耗比例，适用于集群规模较少变动的场景；其分值由节点空闲资源与节点总容量的比值计算而来，即由CPU或内存资源的总容量减去节点上已有Pod对象需求的容量总和，再减去当前要创建的Pod对象的需求容量得到的结果除以总容量；CPU和内存具有相同权重，资源空闲比例越高的节点得分也就越高，其计算公式如为：(cpu((capacity – sum(requested)) * 10 / capacity) + memory((capacity – sum(requested)) * 10 / capacity))/ 2。
	MostRequestedPriority：与优选函数LeastRequestedPriority的评估节点得分的方法相似，但二者不同的是，当前函数将给予计算资源占用比例更大的节点以更高的得分，计算公式如为：(cpu((sum(requested)) * 10 / capacity) + memory((sum(requested)) * 10 / capacity))/ 2。该函数的目标在于优先让节点以满载的方式承载Pod资源，从而能够使用更少的节点数，因而较适用于节点规模可弹性伸缩的集群中以最大化地节约节点数量。
	BalancedResourceAllocation：以CPU和内存资源占用率的相近程度作为评估标准，二者越接近的节点权重越高。该优选函数不能单独使用，它需要和LeastRequestedPriority组合使用来平衡优化节点资源的使用状态，选择那些在部署当前Pod资源后系统资源更为均衡的节点。
	ResourceLimitsPriority：以是否能够满足Pod资源限制为评估标准，那些能够满足Pod对于CPU或（和）内存资源限制的节点将计入1分，节点未声明可分配资源或Pod未定义资源限制时不影响节点计分。
	RequestedToCapacityRatio：该函数允许用户自定义节点各类资源（例如CPU和内存等）的权重，以便提高大型集群中稀缺资源的利用率；该函数的行为可以通过名为requestedToCapacityRatioArguments的配置选项进行控制，它由shape和resources两个参数组成。
	NodeAffinityPriority：节点亲和调度机制，它根据Pod资源规范中的spec.nodeSelector来对给定节点进行匹配度检查，成功匹配到的条目越多则节点得分越高。不过，其评估过程使用PreferredDuringSchedulingIgnoredDuringExecution这一表示首选亲和的标签选择器。
	ImageLocalityPriority：镜像亲和调度机制，它根据给定节点上是否拥有运行当前Pod对象中的容器所依赖到的镜像文件来计算该节点的得分值。那些不具有该Pod对象所依赖到的任何镜像文件的节点得分为0，而那些存在相关镜像文件的各节点中，拥有被Pod所依赖到的镜像文件的体积之和越大的节点得分就会越高。
	TaintTolerationPriority：基于对Pod资源对节点的污点容忍调度偏好进行其优先级评估，它将Pod对象的tolerations列表与节点的污点进行匹配度检查，成功匹配的条目越多，则节点得分越低。
	SelectorSpreadPriority：尽可能分散Pod至不同节点上的调度机制，它首先查找标签选择器能够匹配到当前Pod标签的ReplicationController、ReplicaSet和StatefulSet等控制器对象，而后查找可由这类对象的标签选择器匹配到的现存各Pod对象及其所在的节点，而那些运行此类Pod对象越少的节点得分越高。简单来说，如其名称所示，此优选函数尽量把同一标签选择器匹配到的Pod资源打散到不同的节点上运行。
	ServiceSpreadingPriority：类似于SelectorSpreadPriority，它首先查找标签选择器能够匹配到当前Pod标签的Service对象，而后查找可由这类Service对象的标签选择器匹配到的现存各Pod对象及其所在的节点，而那些运行此类Pod对象越少的节点得分越高。
	EvenPodsSpreadPriority：用于将一组特定的Pod对象在指定的拓扑结构上进行均衡打散，打散条件定义在Pod对象的spec.topologySpreadConstraints字段上，它内嵌labelSelector指定标签选择器以匹配符合条件的Pod对象，使用topologyKey指定目标拓扑结构，使用maxSkew描述最大允许的不均衡数量，而无法满足指定的调度条件时的评估策略则由whenUnsatisfiable字段定义，它有两个可用取值，默认值DoNotSchedule表示不予调度，而ScheduleAnyway则表示以满足最小不均衡值的标准进行调度。
	EqualPriority：设定所有节点具有相同的权重1。
	InterPodAffinityPriority：遍历Pod对象的亲和性条目，并将那些能够匹配到给定节点的条目的权重相加，结果值越大的节点得分越高。
	NodePreferAvoidPodsPriority：此优选级函数权限默认为10000，它根据节点是否设置了注解信息scheduler.alpha.kubernetes.io/preferAvoidPods来计算其优选级。计算方式是，给定的节点无此注解信息时，其得分为10乘以权重10000，存在此注解信息时，对于那些由ReplicationController或ReplicaSet控制器管控的Pod对象的得分为0，其他Pod对象会被忽略（得最高分）。

# kubectl get ippool
NAME                  AGE
america-ippool        15h
default-ipv4-ippool   110d
england-ippool        14h
germany-ippool        3d1h



apiVersion: v1
items:
- apiVersion: crd.projectcalico.org/v1
  kind: IPPool
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{},"name":"america-ippool"},"spec":{"allowedUses":["Workload","Tunnel"],"cidr":"10.200.66.0/24","disabled":false,"ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()"}}
    creationTimestamp: "2023-07-09T11:57:51Z"
    generation: 2
    name: america-ippool
    resourceVersion: "39183913"
    uid: f054302a-3d18-45eb-ae0d-4d685c66619e
  spec:
    allowedUses:
    - Workload
    - Tunnel
    cidr: 10.200.66.0/24
    disabled: false
    ipipMode: Always
    natOutgoing: true
    nodeSelector: all()
- apiVersion: crd.projectcalico.org/v1
  kind: IPPool
  metadata:
    annotations:
      projectcalico.org/metadata: '{"uid":"1ba06317-30af-4017-9665-9e47f1f20a62","creationTimestamp":"2023-03-21T06:17:28Z"}'
    creationTimestamp: "2023-03-21T06:17:28Z"
    generation: 1
    name: default-ipv4-ippool
    resourceVersion: "1281"
    uid: e6ffc1cb-c7a3-4236-9dd6-bf7b7fc3ac92
  spec:
    allowedUses:
    - Workload
    - Tunnel
    blockSize: 24
    cidr: 10.233.64.0/18
    ipipMode: Always
    natOutgoing: true
    nodeSelector: all()
    vxlanMode: Never
- apiVersion: crd.projectcalico.org/v1
  kind: IPPool
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{},"name":"england-ippool"},"spec":{"allowedUses":["Workload","Tunnel"],"cidr":"10.200.65.0/24","disabled":false,"ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()"}}
    creationTimestamp: "2023-07-09T12:34:40Z"
    generation: 2
    name: england-ippool
    resourceVersion: "39184018"
    uid: 7b9e7c9e-3fe2-4999-8f0b-2ebad97294b2
  spec:
    allowedUses:
    - Workload
    - Tunnel
    cidr: 10.200.65.0/24
    disabled: false
    ipipMode: Always
    natOutgoing: true
    nodeSelector: all()
- apiVersion: crd.projectcalico.org/v1
  kind: IPPool
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"crd.projectcalico.org/v1","kind":"IPPool","metadata":{"annotations":{},"name":"germany-ippool"},"spec":{"allowedUses":["Workload","Tunnel"],"cidr":"10.66.141.1/24","disabled":false,"ipipMode":"CrossSubnet","natOutgoing":true,"nodeSelector":"all()"}}
    creationTimestamp: "2023-07-07T02:08:56Z"
    generation: 9
    name: germany-ippool
    resourceVersion: "39184084"
    uid: d1028fc8-1960-40ef-8f4a-0266606fb360
  spec:
    allowedUses:
    - Workload
    - Tunnel
    cidr: 10.66.141.1/24
    disabled: false
    ipipMode: Always
    natOutgoing: true
    nodeSelector: all()
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""