# kubectl get configmaps -n kube-system 
NAME                                 DATA   AGE
...
kube-proxy                           2      2d16h # 修改此项configmap
...


# kubectl edit configmaps -n kube-system kube-proxy
...
    iptables:
      masqueradeAll: false
      masqueradeBit: null
      minSyncPeriod: 0s
      syncPeriod: 0s
    ipvs:
      excludeCIDRs: null
      minSyncPeriod: 0s
      scheduler: "" # 调度算法，默认轮询
      strictARP: false
      syncPeriod: 0s
      tcpFinTimeout: 0s
      tcpTimeout: 0s
      udpTimeout: 0s
    kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: "ipvs" # 默认为空，表示使用的是iptables代理模式
...


# 保存退出后显示次结果表示修改成功
configmap/kube-proxy edited


# kube-proxy 需重建后才能生效
# kubectl get pod -n kube-system 
NAME                                   READY   STATUS    RESTARTS      AGE
...
kube-proxy-79gw9                       1/1     Running   3 (42h ago)   2d15h
kube-proxy-dr7l5                       1/1     Running   2 (42h ago)   2d16h
...


# 重建 kube-proxy，生产环境修改的话需要一个一个删除然后由k8s自动重建，这里采用标签选择的方式全部删除重建
# 获取标签
# kubectl get pod -n kube-system --show-labels
...
# 全部重建
# kubectl delete pod -n kube-system  -l k8s-app=kube-proxy
pod "kube-proxy-79gw9" deleted
pod "kube-proxy-dr7l5" deleted
# 查看是否重建成功
# kubectl get pod -n kube-system  -l k8s-app=kube-proxy
NAME               READY   STATUS    RESTARTS   AGE
kube-proxy-l9qnk   1/1     Running   0          13s
kube-proxy-wpjtz   1/1     Running   0          12s


# 生成的接口
# ifconfig kube-ipvs0
kube-ipvs0: flags=130<BROADCAST,NOARP>  mtu 1500
        inet 10.96.0.10  netmask 255.255.255.255  broadcast 0.0.0.0
        ether be:6f:c3:98:dc:d1  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0


# 选择一个node节点查看ipvs规则生成情况
root@k8s-node-1:~# apt-get install ipvsadm
...
# node节点情况
root@k8s-node-1:~# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.96.0.1:443 rr
  -> 10.0.0.100:6443              Masq    1      0          0         
TCP  10.96.0.10:53 rr
  -> 10.244.0.6:53                Masq    1      0          0         
  -> 10.244.0.7:53                Masq    1      0          0         
TCP  10.96.0.10:9153 rr
  -> 10.244.0.6:9153              Masq    1      0          0         
  -> 10.244.0.7:9153              Masq    1      0          0         
UDP  10.96.0.10:53 rr
  -> 10.244.0.6:53                Masq    1      0          0         
  -> 10.244.0.7:53                Masq    1      0          0         

# 控制端查看service
# kubectl get service -A
NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP                  2d16h
kube-system   kube-dns     ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP,9153/TCP   2d16h

kubectl get configmap coredns -n kube-system -o yaml | grep kubernetes | awk '{print $2}'

apiVersion: v1
kind: Service
metadata:
  name: …
  namespace: …
  labels:
    key1: value1
    key2: value2
spec:
  type <string>   # Service类型，默认为ClusterIP
  selector <map[string]string>  # 等值类型的标签选择器，内含“与”逻辑
                                # 注意：Service 资源的 selector 只有基于等式的
  ports：  # Service的端口对象列表
  - name <string>  # 端口名称
    protocol <string>  # 协议，目前仅支持TCP、UDP和SCTP，默认为TCP
    port <integer>  # Service的端口号
    targetPort  <string>  # 后端目标进程的端口号或名称，名称需由Pod规范定义
    nodePort <integer>  # 节点端口号，仅适用于NodePort和LoadBalancer类型
  clusterIP  <string>  # Service的集群IP，建议由系统自动分配
  externalTrafficPolicy  <string> # 外部流量策略处理方式，Local表示由当前节点处理，Cluster表示向集群范围调度，默认为Cluster，注意：如果选择为Local 则当当前节点pod宕机时访问可能出现问题
  loadBalancerIP  <string>  # 外部负载均衡器使用的IP地址，仅适用于LoadBlancer，即只能对接Iaas平台提供的负载均衡
  externalName <string>  # 外部服务名称，该名称将作为Service的DNS CNAME值

apiVersion: v1
kind: Service
metadata:
  namespace: app1
  name: app1-nginx
  labels:
    app: app1-nginx
spec:
  selector:
    # 同时满足以下标签的Pod会成为该service的后端
    app: app1-nginx-deployment-label
    version: v1
...

apiVersion: v1
kind: Service
metadata:
  namespace: app1
  name: app1-nginx
  labels:
    app: app1-nginx
spec:
  selector:
    app: app1-nginx-deployment-label
    version: v1
  type: ClusterIP # ClusterIP、NodePort、ExternalName、LoadBalancer
...

kind: Service
apiVersion: v1
metadata:
  name: demoapp-svc
  namespace: default
spec:
  type: ClusterIP # 通常无需指定因为默认既是ClusterIP
  clusterIP: 10.97.72.1 # ClusterIP 的 IP 可以自定义，但通常由 K8S 动态分配
  selector:
    app: demoapp
  ports:
  - name: http
    protocol: TCP
    port: 80
    targetPort: 80

kind: Service
apiVersion: v1
metadata:
  name: demoapp-nodeport-svc
spec:
  type: NodePort
  clusterIP: 10.97.56.1
  selector:
    app: demoapp
  ports:
  - name: http
    protocol: TCP
    port: 80 # service的端口，集群内部提供服务
    targetPort: 80
    nodePort: 30080 # nodePort的端口，集群外部提供服务
  # externalTrafficPolicy: Local

kind: Service
apiVersion: v1
metadata:
  name: demoapp-loadbalancer-svc
spec:
  type: LoadBalancer
  selector:
    app: demoapp
  ports:
  - name: http
    protocol: TCP
    port: 80
    targetPort: 80
  loadBalancerIP: 1.2.3.4

kind: Service
apiVersion: v1
metadata:
  name: externalname-redis-svc
  namespace: default
spec:
  type: ExternalName
  externalName: redis.ik8s.io
  ports:
  - protocol: TCP
    port: 6379
    targetPort: 6379
    nodePort: 0
  selector: {}

apiVersion: v1
kind: Service
metadata:
  name: my-service
spec:
  type: ExternalName
  externalName: my.external.domain.com

apiVersion: v1
kind: Service
metadata:
  name: mysql-external
spec:
  type: ExternalName
  externalName: mysql.external.com

const mysql = require('mysql');
const connection = mysql.createConnection({
  host: 'mysql-external',
  user: 'root',
  password: 'password',
  database: 'mydatabase'
});

apiVersion: v1
kind: Service
...
spec:
...
  clusterIP: None # Headless Service，因为 type 默认就是 ClusterIP，所以不用指定
...

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: demoapp
  template:
    metadata:
      labels:
        app: demoapp
    spec:
      containers:
      - name: demoapp
        image: ikubernetes/demoapp:v1.0
---
apiVersion: v1
kind: Service
metadata:
  name: demoapp-svc
spec:
  type: ClusterIP
  selector:
    app: demoapp
  ports:
  - port: 80
    targetPort: 80

# kubectl describe svc demoapp-svc 
Name:              demoapp-svc
Namespace:         default
Labels:            <none>
Annotations:       <none>
Selector:          app=demoapp
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                10.100.31.12 # 分配的ClusterIP
IPs:               10.100.31.12
Port:              <unset>  80/TCP
TargetPort:        80/TCP
Endpoints:         10.244.1.17:80,10.244.1.18:80,10.244.1.19:80
Session Affinity:  None
Events:            <none>


# kubectl exec demoapp-5748b7ccfc-22hlj -- nslookup demoapp-svc.default.svc.k8s.xiangzheng.com
Server:		10.96.0.10
Address:	10.96.0.10#53

Name:	demoapp-svc.default.svc.k8s.xiangzheng.com
Address: 10.100.31.12 # 解析出来的是service的ClusterIP

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: demoapp
  template:
    metadata:
      labels:
        app: demoapp
    spec:
      containers:
      - name: demoapp
        image: ikubernetes/demoapp:v1.0
---
apiVersion: v1
kind: Service
metadata:
  name: demoapp-svc
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: demoapp
  ports:
  - port: 80
    targetPort: 80

# kubectl describe svc demoapp-svc 
Name:              demoapp-svc
Namespace:         default
Labels:            <none>
Annotations:       <none>
Selector:          app=demoapp
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                None # 不会分配serviceIP
IPs:               None
Port:              <unset>  80/TCP
TargetPort:        80/TCP
Endpoints:         10.244.1.20:80,10.244.1.21:80,10.244.1.22:80
Session Affinity:  None
Events:            <none>


# 解析出来的是pod的真实IP
# kubectl exec demoapp-5748b7ccfc-bgzsj -- nslookup demoapp-svc.default.svc.k8s.xiangzheng.com
Server:		10.96.0.10
Address:	10.96.0.10#53

Name:	demoapp-svc.default.svc.k8s.xiangzheng.com
Address: 10.244.1.21
Name:	demoapp-svc.default.svc.k8s.xiangzheng.com
Address: 10.244.1.20
Name:	demoapp-svc.default.svc.k8s.xiangzheng.com
Address: 10.244.1.22



# kubectl exec demoapp-5748b7ccfc-bgzsj -- curl demoapp-svc.default.svc.k8s.xiangzheng.com -s
10.244.1.20, ServerName: demoapp-5748b7ccfc-bgzsj, ServerIP: 10.244.1.20!
# kubectl exec demoapp-5748b7ccfc-bgzsj -- curl demoapp-svc.default.svc.k8s.xiangzheng.com -s
10.244.1.20, ServerName: demoapp-5748b7ccfc-qkntg, ServerIP: 10.244.1.21!
# kubectl exec demoapp-5748b7ccfc-bgzsj -- curl demoapp-svc.default.svc.k8s.xiangzheng.com -s
10.244.1.20, ServerName: demoapp-5748b7ccfc-ql8sf, ServerIP: 10.244.1.22!
# kubectl exec demoapp-5748b7ccfc-bgzsj -- curl demoapp-svc.default.svc.k8s.xiangzheng.com -s
10.244.1.20, ServerName: demoapp-5748b7ccfc-ql8sf, ServerIP: 10.244.1.22!

apiVersion: v1
kind: Service
...
spec:
...
  ports:
  - name: http
    protocol: HTTP # 协议，目前仅支持TCP、UDP和SCTP，默认为TCP
    port: 80 # service的80端口
    targetPort: 80 # 目标pod内部的端口，默认情况下，targetPort 将被设置为与 port 字段相同的值。
    nodePort: 30080 # 自定义node节点对外暴露的端口，默认为30000-32767之间

kind: Service
apiVersion: v1
metadata:
  namespace: app1
  name: app1-nginx
  labels:
    app: app1-nginx-service
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
    nodePort: 30080
  # 定义多个
  - name: https
    port: 443
    protocol: TCP
    targetPort: 443
    nodePort: 30043
  selector:
    app: app1-nginx-deployment-label
    version: v1

kind: Service
apiVersion: v1
metadata:
  name: demoapp-svc
  namespace: default
spec:
  type: ClusterIP # 也可以定义为 NodePort 进而实现外部访问 externalIP 或 NodePort 都可以
                  # 注意：在设置 externalIPs 后 type: 不能为 LoadBalancer
  selector:
    app: demoapp
  ports:
  - name: http
    protocol: TCP
    port: 80
    targetPort: 80
  externalIPs: # 列表格式，可以定义多个
  - '10.0.0.168'

# kubectl describe svc demoapp-svc 
Name:              demoapp-svc
Namespace:         default
Labels:            <none>
Annotations:       <none>
Selector:          app=demoapp
Type:              ClusterIP
IP Family Policy:  SingleStack
IP Families:       IPv4
IP:                10.109.71.156
IPs:               10.109.71.156
External IPs:      10.0.0.168
Port:              http  80/TCP
TargetPort:        80/TCP
Endpoints:         10.244.1.91:80,10.244.1.92:80,10.244.2.35:80
Session Affinity:  None
Events:            <none>


# curl 10.0.0.168
iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-65bc49b76-cj6lw, ServerIP: 10.244.1.92!


# 并且每个节点上都会自动生成10.0.0.168的IP以及80端口（80端口对应的是service的80端口）
#root@k8s-master-1:~# ss -ntl | grep 168 ; ip a | grep 168
LISTEN  0        4096          10.0.0.168:80             0.0.0.0:*              
    inet 10.0.0.168/32 scope global kube-ipvs0
#root@k8s-node-1:~# ss -ntl | grep 168 ; ip a | grep 168
LISTEN  0        4096          10.0.0.168:80             0.0.0.0:*              
    inet 10.0.0.168/32 scope global kube-ipvs0
#root@k8s-node-2:~# ss -ntl | grep 168 ; ip a | grep 168
LISTEN  0        4096          10.0.0.168:80             0.0.0.0:*              
    inet 10.0.0.168/32 scope global kube-ipvs0

# vim recommended.yaml
...
---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard
  externalIPs: # 列表格式，可以定义多个
  - '10.0.0.168'
---
...

# kubectl get svc -n kubernetes-dashboard kubernetes-dashboard 
NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
kubernetes-dashboard   ClusterIP   10.102.14.235   10.0.0.168    443/TCP   5m40s


# kubectl get ep -n kubernetes-dashboard kubernetes-dashboard 
NAME                   ENDPOINTS           AGE
kubernetes-dashboard   10.244.1.220:8443   6m3s



# 任何node节点都会生成ipvs规则（前提使用的是ipvs代理模式，而非iptables）
# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.168:443 rr
  -> 10.244.1.220:8443            Masq    1      0          0         
...

apiVersion: v1
kind: Service
...
spec:
...
  sessionAffinity: None

kind: Service
apiVersion: v1
metadata:
  name: ${NAME}
  namespace: ${NAMESPACE}
spec:
  type: NodePort
  selector:
    ${SELECT_KEY}: ${SELECT_VALUE}
  ports:
  - name: http
    protocol: TCP
    port: ${TARGET_PORT} # service的端口，集群内部提供服务
    targetPort: ${TARGET_PORT} # 容器自身监听的端口
    nodePort: ${EXTERNAL_PORT} # nodePort的端口，集群外部提供服务

NAMESPACE="england-10-200-66-1-24" \
NAME="client" \
SELECT_KEY="run" \
SELECT_VALUE="client" \
TARGET_PORT=19001 \
EXTERNAL_PORT=32222 \
envsubst < service-tmp.yaml | kubectl apply -f -