kubeasz Documentation

roles/calico/
├── tasks
│   └── main.yml
├── templates
│   ├── calico-csr.json.j2
│   ├── calicoctl.cfg.j2
│   ├── calico-v3.15.yaml.j2
│   ├── calico-v3.19.yaml.j2
│   └── calico-v3.8.yaml.j2
└── vars
    └── main.yml

{
  "CN": "calico",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
  datastoreType: "etcdv3"
  etcdEndpoints: {{ ETCD_ENDPOINTS }}
  etcdKeyFile: /etc/calico/ssl/calico-key.pem
  etcdCertFile: /etc/calico/ssl/calico.pem
  etcdCACertFile: {{ ca_dir }}/ca.pem

kubectl get pod --all-namespaces
NAMESPACE     NAME                                       READY     STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-5c6b98d9df-xj2n4   1/1       Running   0          1m
kube-system   calico-node-4hr52                          2/2       Running   0          1m
kube-system   calico-node-8ctc2                          2/2       Running   0          1m
kube-system   calico-node-9t8md                          2/2       Running   0          1m

# 查看网卡信息
ip a

# 查看路由
route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.1.1     0.0.0.0         UG    0      0        0 ens3
192.168.1.0     0.0.0.0         255.255.255.0   U     0      0        0 ens3
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
172.20.3.64     192.168.1.34    255.255.255.192 UG    0      0        0 ens3
172.20.33.128   0.0.0.0         255.255.255.192 U     0      0        0 *
172.20.33.129   0.0.0.0         255.255.255.255 UH    0      0        0 caliccc295a6d4f
172.20.104.0    192.168.1.35    255.255.255.192 UG    0      0        0 ens3
172.20.166.128  192.168.1.63    255.255.255.192 UG    0      0        0 ens3

calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+--------------+-------------------+-------+----------+-------------+
| 192.168.1.34 | node-to-node mesh | up    | 12:34:00 | Established |
| 192.168.1.35 | node-to-node mesh | up    | 12:34:00 | Established |
| 192.168.1.63 | node-to-node mesh | up    | 12:34:01 | Established |
+--------------+-------------------+-------+----------+-------------+

netstat -antlp|grep ESTABLISHED|grep 179
tcp        0      0 192.168.1.66:179        192.168.1.35:41316      ESTABLISHED 28479/bird      
tcp        0      0 192.168.1.66:179        192.168.1.34:40243      ESTABLISHED 28479/bird      
tcp        0      0 192.168.1.66:179        192.168.1.63:48979      ESTABLISHED 28479/bird

# 查看所有calico相关数据
ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico
# 查看 calico网络为各节点分配的网段
ETCDCTL_API=3 etcdctl --endpoints="http://127.0.0.1:2379" get --prefix /calico/ipam/v2/host

$ kubectl get node
NAME           STATUS                     ROLES    AGE    VERSION
192.168.1.1   Ready,SchedulingDisabled   master   178m   v1.13.1
192.168.1.2   Ready,SchedulingDisabled   master   178m   v1.13.1
192.168.1.3   Ready                      node     178m   v1.13.1
192.168.1.4   Ready                      node     178m   v1.13.1
$ kubectl get pod -n kube-system -o wide | grep calico
calico-kube-controllers-77487546bd-jqrlc   1/1     Running   0          179m   192.168.1.3   192.168.1.3   <none>           <none>
calico-node-67t5m                          2/2     Running   0          179m   192.168.1.1   192.168.1.1   <none>           <none>
calico-node-drmhq                          2/2     Running   0          179m   192.168.1.2   192.168.1.2   <none>           <none>
calico-node-rjtkv                          2/2     Running   0          179m   192.168.1.4   192.168.1.4   <none>           <none>
calico-node-xtspl                          2/2     Running   0          179m   192.168.1.3   192.168.1.3   <none>           <none>

/* Detailed source-code truncated for AI context efficiency. */

$ calicoctl get node -o wide
NAME     ASN       IPV4              IPV6
k8s401   (64512)   192.168.1.1/24
k8s402   (64512)   192.168.1.2/24
k8s403   (64512)   192.168.1.3/24
k8s404   (64512)   192.168.1.4/24

#配置routeReflectorClusterID
calicoctl patch node k8s401 -p '{"spec": {"bgp": {"routeReflectorClusterID": "244.0.0.1"}}}'

#配置node label
calicoctl patch node k8s401 -p '{"metadata": {"labels": {"route-reflector": "true"}}}'

$ cat << EOF | calicoctl create -f -
kind: BGPPeer
apiVersion: projectcalico.org/v3
metadata:
  name: peer-with-route-reflectors
spec:
  nodeSelector: all()
  peerSelector: route-reflector == 'true'
EOF

$ cat << EOF | calicoctl create -f -
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: false
  asNumber: 64512
EOF

/* Detailed source-code truncated for AI context efficiency. */

/* Detailed source-code truncated for AI context efficiency. */

cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:         OK
 \__/¯¯\__/    Operator:       OK
 /¯¯\__/¯¯\    Hubble:         OK
 \__/¯¯\__/    ClusterMesh:    disabled
    \__/

DaemonSet         cilium             Desired: 4, Ready: 4/4, Available: 4/4
Deployment        cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
Deployment        hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1
Deployment        hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1
Containers:       cilium             Running: 4
                  cilium-operator    Running: 1
                  hubble-relay       Running: 1
                  hubble-ui          Running: 1
Cluster Pods:     17/17 managed by Cilium
Image versions    hubble-relay       easzlab.io.local:5000/cilium/hubble-relay:v1.11.6: 1
                  hubble-ui          easzlab.io.local:5000/cilium/hubble-ui:v0.9.0: 1
                  hubble-ui          easzlab.io.local:5000/cilium/hubble-ui-backend:v0.9.0: 1
                  cilium             easzlab.io.local:5000/cilium/cilium:v1.11.6: 4
                  cilium-operator    easzlab.io.local:5000/cilium/operator-generic:v1.11.6: 1

$ kubectl get pods,svc
NAME                             READY     STATUS    RESTARTS   AGE
pod/deathstar-5fc7c7795d-djf2q   1/1       Running   0          4h
pod/deathstar-5fc7c7795d-hrgst   1/1       Running   0          4h
pod/tiefighter                   1/1       Running   0          4h
pod/xwing                        1/1       Running   0          4h

NAME                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
service/deathstar    ClusterIP   10.68.242.130   <none>        80/TCP    4h
service/kubernetes   ClusterIP   10.68.0.1       <none>        443/TCP   5h

/* Detailed source-code truncated for AI context efficiency. */

$ kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed # 成功着陆
$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed # 成功着陆

$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed # 成功着陆

$ kubectl exec xwing -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
# 失败超时

/* Detailed source-code truncated for AI context efficiency. */

$ kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port
Panic: deathstar exploded

goroutine 1 [running]:
main.HandleGarbage(0x2080c3f50, 0x2, 0x4, 0x425c0, 0x5, 0xa)
        /code/src/github.com/empire/deathstar/
        temp/main.go:9 +0x64
main.main()
        /code/src/github.com/empire/deathstar/
        temp/main.go:5 +0x85

$ kubectl exec tiefighter -- curl -s -XPOST deathstar.default.svc.cluster.local/v1/request-landing
Ship landed
$ kubectl exec tiefighter -- curl -s -XPUT deathstar.default.svc.cluster.local/v1/exhaust-port
Access denied

$ kubectl exec -n kube-system cilium-6t5vx -- cilium policy get
[
  {
    "endpointSelector": {
      "matchLabels": {
        "any:class": "deathstar",
        "any:org": "empire",
        "k8s:io.kubernetes.pod.namespace": "default"
      }
    },
    "ingress": [
      {
        "fromEndpoints": [
          {
            "matchLabels": {
              "any:org": "empire",
              "k8s:io.kubernetes.pod.namespace": "default"
            }
          }
        ],
        "toPorts": [
          {
            "ports": [
              {
                "port": "80",
                "protocol": "TCP"
              }
            ],
            "rules": {
              "http": [
                {
                  "path": "/v1/request-landing",
                  "method": "POST"
                }
              ]
            }
          }
        ]
      }
    ],
    "labels": [
      {
        "key": "io.cilium.k8s.policy.name",
        "value": "rule1",
        "source": "k8s"
      },
      {
        "key": "io.cilium.k8s.policy.namespace",
        "value": "default",
        "source": "k8s"
      }
    ]
  }
]
Revision: 267

FLANNEL_NETWORK=10.1.0.0/16
FLANNEL_SUBNET=10.1.17.1/24
FLANNEL_MTU=1472
FLANNEL_IPMASQ=true

{
	"name": "mynet",
	"type": "bridge",
	"mtu": 1472,
	"ipMasq": false,
	"isGateway": true,
	"ipam": {
		"type": "host-local",
		"subnet": "10.1.17.0/24"
	}
}

# kubectl get pod --all-namespaces
NAMESPACE     NAME                    READY     STATUS    RESTARTS   AGE
kube-system   kube-flannel-ds-m8mzm   1/1       Running   0          3m
kube-system   kube-flannel-ds-mnj6j   1/1       Running   0          3m
kube-system   kube-flannel-ds-mxn6k   1/1       Running   0          3m

# kubectl get pod --all-namespaces -o wide|head -n 4
NAMESPACE     NAME                    READY     STATUS    RESTARTS   AGE       IP             NODE
default       busy-5956b54c8b-ld4gb   1/1       Running   0          9m        172.20.2.7     192.168.1.1
default       busy-5956b54c8b-lj9l9   1/1       Running   0          9m        172.20.1.5     192.168.1.2
default       busy-5956b54c8b-wwpkz   1/1       Running   0          9m        172.20.0.6     192.168.1.3

# 查看路由
# ip route
default via 192.168.1.254 dev ens3 onlink 
192.168.1.0/24 dev ens3  proto kernel  scope link  src 192.168.1.1 
172.17.0.0/16 dev docker0  proto kernel  scope link  src 172.17.0.1 linkdown 
172.20.0.0/24 via 192.168.1.3 dev ens3 
172.20.1.0/24 via 192.168.1.2 dev ens3 
172.20.2.0/24 dev cni0  proto kernel  scope link  src 172.20.2.1

ping 172.20.2.7
ping 172.20.1.5
ping 172.20.0.6

roles/kube-ovn
├── defaults
│   └── main.yml		# kube-ovn 相关配置文件
├── tasks
│   └── main.yml		# 安装执行文件
└── templates
    ├── crd.yaml.j2	        # crd 模板
    ├── kube-ovn.yaml.j2	# kube-ovn yaml 模板
    └── ovn.yaml.j2		    # ovn yaml 模板

/* Detailed source-code truncated for AI context efficiency. */

# 创建一个 namespace: test-ns
$ cat > test-ns.yaml << EOF
apiVersion: v1
kind: Namespace
metadata:
  annotations:
  name: test-ns
EOF
$ kubectl apply -f test-ns.yaml

# 创建一个 subnet: test-subnet 并绑定 namespace test-ns
$ cat > test-subnet.yaml << EOF
apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
  name: test-subnet
spec:
  protocol: IPv4
  default: false
  namespaces:
  - test-ns
  cidrBlock: 10.17.0.0/24
  gateway: 10.17.0.1
  excludeIps:
  - 10.17.0.1..10.17.0.10
EOF
$ kubectl apply -f test-subnet.yaml

# 在 test-ns 中创建 nginx 部署
$ kubectl run -n test-ns nginx --image=nginx --replicas=2 --port=80 --expose

# 在 default 中创建 busy 客户端
$ kubectl run busy --image=busybox sleep 360000

/* Detailed source-code truncated for AI context efficiency. */

# master上路由
root@master1:~$ ip route
...
172.20.1.0/24 via 192.168.1.2 dev ens3  proto 17 
172.20.2.0/24 via 192.168.1.3 dev ens3  proto 17 
...

# node3上路由
root@node3:~$ ip route
... 
172.20.0.0/24 via 192.168.1.1 dev ens3  proto 17 
172.20.1.0/24 via 192.168.1.2 dev ens3  proto 17 
172.20.2.0/24 dev kube-bridge  proto kernel  scope link  src 172.20.2.1 
...

# master上
root@master1:~$ netstat -antlp|grep router|grep LISH|grep 179
tcp        0      0 192.168.1.1:179        192.168.1.3:58366      ESTABLISHED 26062/kube-router
tcp        0      0 192.168.1.1:42537      192.168.1.2:179        ESTABLISHED 26062/kube-router

# node3上
root@node3:~$ netstat -antlp|grep router|grep LISH|grep 179
tcp        0      0 192.168.1.3:58366      192.168.1.1:179        ESTABLISHED 18897/kube-router
tcp        0      0 192.168.1.3:179        192.168.1.2:43928      ESTABLISHED 18897/kube-router

$ ipset list
...
Name: kube-router-pod-subnets
Type: hash:net
Revision: 6
Header: family inet hashsize 1024 maxelem 65536 timeout 0
Size in memory: 672
References: 2
Members:
172.20.1.0/24 timeout 0
172.20.2.0/24 timeout 0
172.20.0.0/24 timeout 0

Name: kube-router-node-ips
Type: hash:ip
Revision: 4
Header: family inet hashsize 1024 maxelem 65536 timeout 0
Size in memory: 416
References: 1
Members:
192.168.1.1 timeout 0
192.168.1.2 timeout 0
192.168.1.3 timeout 0
...

kubectl get cronjobs.batch -n network-test
NAME                                  SCHEDULE      SUSPEND   ACTIVE   LAST SCHEDULE   AGE
test01-pod-to-container               */5 * * * *   False     0        3m19s           6d3h
test02-pod-to-node-nodeport           */5 * * * *   False     0        3m19s           6d3h
test03-pod-to-multi-node-clusterip    */5 * * * *   False     1        6d3h            6d3h
test04-pod-to-multi-node-headless     */5 * * * *   False     1        6d3h            6d3h
test05-pod-to-multi-node-nodeport     */5 * * * *   False     1        6d3h            6d3h
test06-pod-to-external-1111           */5 * * * *   False     0        3m19s           6d3h
test07-pod-to-external-fqdn-baidu     */5 * * * *   False     0        3m19s           6d3h
test08-host-to-multi-node-clusterip   */5 * * * *   False     1        6d3h            6d3h
test09-host-to-multi-node-headless    */5 * * * *   False     1        6d3h            6d3h

# network-check 自动安装
network_check_enabled: true
network_check_schedule: "*/5 * * * *"  # 检测频率，默认5分钟执行一次

kubectl get pod -n network-test
NAME                                                 READY   STATUS      RESTARTS   AGE
echo-server-58d7bb7f6-77ps6                          1/1     Running     0          6d4h
echo-server-host-cc87c966d-bk57t                     1/1     Running     0          6d4h
test01-pod-to-container-27606775-q6xlb               0/1     Completed   0          3m10s
test02-pod-to-node-nodeport-27606775-x2v5d           0/1     Completed   0          3m10s
test03-pod-to-multi-node-clusterip-27597895-cbq8d    0/1     Pending     0          6d4h
test04-pod-to-multi-node-headless-27597895-qzsgz     0/1     Pending     0          6d4h
test05-pod-to-multi-node-nodeport-27597895-kb5r7     0/1     Pending     0          6d4h
test06-pod-to-external-1111-27606775-p6v8s           0/1     Completed   0          3m10s
test07-pod-to-external-fqdn-baidu-27606775-qdfwd     0/1     Completed   0          3m10s
test08-host-to-multi-node-clusterip-27597895-qsgn9   0/1     Pending     0          6d4h
test09-host-to-multi-node-headless-27597895-hpkt5    0/1     Pending     0          6d4h

kubectl delete ns network-test

#$IP为所有节点地址包括自身，按照提示输入yes 和root密码
ssh-copy-id $IP

export release=3.5.0
wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown

# 国内环境
./ezdown -D
# 海外环境
#./ezdown -D -m standard

# 按需下载
./ezdown -X flannel
./ezdown -X prometheus
...

./ezdown -P

# 容器化运行kubeasz
./ezdown -S

# 创建新集群 k8s-01
docker exec -it kubeasz ezctl new k8s-01
2021-01-19 10:48:23 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-01
2021-01-19 10:48:23 DEBUG set version of common plugins
2021-01-19 10:48:23 DEBUG cluster k8s-01: files successfully created.
2021-01-19 10:48:23 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-01/hosts'
2021-01-19 10:48:23 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-01/config.yml'

#建议使用alias命令，查看~/.bashrc 文件应该包含：alias dk='docker exec -it kubeasz'
source ~/.bashrc

# 一键安装，等价于执行docker exec -it kubeasz ezctl setup k8s-01 all
dk ezctl setup k8s-01 all

# 或者分步安装，具体使用 dk ezctl help setup 查看分步安装帮助信息
# dk ezctl setup k8s-01 01
# dk ezctl setup k8s-01 02
# dk ezctl setup k8s-01 03
# dk ezctl setup k8s-01 04
...

{
  "signing": {
    "default": {
      "expiry": "{{ CERT_EXPIRY }}"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "{{ CERT_EXPIRY }}"
      },
      "kcfg": {
        "usages": [
            "signing",
            "key encipherment",
            "client auth"
        ],
        "expiry": "{{ CUSTOM_EXPIRY }}"
      }
    }
  }
}

{
  "CN": "kubernetes-ca",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}

$ kubectl describe clusterrolebinding cluster-admin
Name:         cluster-admin
Labels:       kubernetes.io/bootstrapping=rbac-defaults
Annotations:  rbac.authorization.kubernetes.io/autoupdate=true
Role:
  Kind:  ClusterRole
  Name:  cluster-admin
Subjects:
  Kind   Name            Namespace
  ----   ----            ---------
  Group  system:masters

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=127.0.0.1:8443
kubectl config set-credentials admin --client-certificate=admin.pem --embed-certs=true --client-key=admin-key.pem
kubectl config set-context kubernetes --cluster=kubernetes --user=admin
kubectl config use-context kubernetes

{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

$ kubectl describe clusterrolebinding system:node-proxier
Name:         system:node-proxier
Labels:       kubernetes.io/bootstrapping=rbac-defaults
Annotations:  rbac.authorization.kubernetes.io/autoupdate=true
Role:
  Kind:  ClusterRole
  Name:  system:node-proxier
Subjects:
  Kind  Name               Namespace
  ----  ----               ---------
  User  system:kube-proxy

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=127.0.0.1:8443 --kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --embed-certs=true --client-key=kube-proxy-key.pem --kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

{
  "CN": "etcd",
  "hosts": [
{% for host in groups['etcd'] %}
    "{{ host }}",
{% endfor %}
    "127.0.0.1"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory={{ ETCD_DATA_DIR }}
ExecStart={{ bin_dir }}/etcd \
  --name=etcd-{{ inventory_hostname }} \
  --cert-file={{ ca_dir }}/etcd.pem \
  --key-file={{ ca_dir }}/etcd-key.pem \
  --peer-cert-file={{ ca_dir }}/etcd.pem \
  --peer-key-file={{ ca_dir }}/etcd-key.pem \
  --trusted-ca-file={{ ca_dir }}/ca.pem \
  --peer-trusted-ca-file={{ ca_dir }}/ca.pem \
  --initial-advertise-peer-urls=https://{{ inventory_hostname }}:2380 \
  --listen-peer-urls=https://{{ inventory_hostname }}:2380 \
  --listen-client-urls=https://{{ inventory_hostname }}:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://{{ inventory_hostname }}:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster={{ ETCD_NODES }} \
  --initial-cluster-state={{ CLUSTER_STATE }} \
  --data-dir={{ ETCD_DATA_DIR }} \
  --wal-dir={{ ETCD_WAL_DIR }} \
  --snapshot-count=50000 \
  --auto-compaction-retention=1 \
  --auto-compaction-mode=periodic \
  --max-request-bytes=10485760 \
  --quota-backend-bytes=8589934592
Restart=always
RestartSec=15
LimitNOFILE=65536
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

/* Detailed source-code truncated for AI context efficiency. */

# 测试示例
mkdir test-data
fio --rw=write --ioengine=sync --fdatasync=1 --directory=test-data --size=2200m --bs=2300 --name=mytest

cat playbooks/04.kube-master.yml
- hosts: kube_master
  roles:
  - kube-lb        # 四层负载均衡，监听在127.0.0.1:6443，转发到真实master节点apiserver服务
  - kube-master    #
  - kube-node      # 因为网络、监控等daemonset组件，master节点也推荐安装kubelet和kube-proxy服务
  ...

{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
{% if groups['ex_lb']|length > 0 %}
    "{{ hostvars[groups['ex_lb'][0]]['EX_APISERVER_VIP'] }}",
{% endif %}
{% for host in groups['kube_master'] %}
    "{{ host }}",
{% endfor %}
    "{{ CLUSTER_KUBERNETES_SVC_IP }}",
{% for host in MASTER_CERT_HOSTS %}
    "{{ host }}",
{% endfor %}
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

/* Detailed source-code truncated for AI context efficiency. */

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart={{ bin_dir }}/kube-controller-manager \
  --allocate-node-cidrs=true \
  --authentication-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --authorization-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --bind-address=0.0.0.0 \
  --cluster-cidr={{ CLUSTER_CIDR }} \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file={{ ca_dir }}/ca.pem \
  --cluster-signing-key-file={{ ca_dir }}/ca-key.pem \
  --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --leader-elect=true \
  --node-cidr-mask-size={{ NODE_CIDR_LEN }} \
  --root-ca-file={{ ca_dir }}/ca.pem \
  --service-account-private-key-file={{ ca_dir }}/ca-key.pem \
  --service-cluster-ip-range={{ SERVICE_CIDR }} \
  --use-service-account-credentials=true \
  --v=2
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart={{ bin_dir }}/kube-scheduler \
  --authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
  --authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
  --bind-address=0.0.0.0 \
  --kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
  --leader-elect=true \
  --v=2
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target

# 查看进程状态
systemctl status kube-apiserver
systemctl status kube-controller-manager
systemctl status kube-scheduler
# 查看进程运行日志
journalctl -u kube-apiserver
journalctl -u kube-controller-manager
journalctl -u kube-scheduler

NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok                   
controller-manager   Healthy   ok                   
etcd-0               Healthy   {"health": "true"}   
etcd-2               Healthy   {"health": "true"}   
etcd-1               Healthy   {"health": "true"}

cat playbooks/05.kube-node.yml
- hosts: kube_node
  roles:
  - { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
  - { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }

/* Detailed source-code truncated for AI context efficiency. */

[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart={{ bin_dir }}/kube-proxy \
  --config=/var/lib/kube-proxy/kube-proxy-config.yaml
Restart=always
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

systemctl status kubelet	# 查看状态
systemctl status kube-proxy
journalctl -u kubelet		# 查看日志
journalctl -u kube-proxy

NAME           STATUS    ROLES     AGE       VERSION
192.168.1.42   Ready     <none>    2d        v1.9.0
192.168.1.43   Ready     <none>    2d        v1.9.0
192.168.1.44   Ready     <none>    2d        v1.9.0

apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-es-0
spec:
  capacity:
    storage: 4Gi
  accessModes:
    - ReadWriteMany
  volumeMode: Filesystem
  persistentVolumeReclaimPolicy: Recycle
  storageClassName: "es-storage-class"
  nfs:
    # 根据实际共享目录修改
    path: /share/es0
    # 根据实际 nfs服务器地址修改
    server: 192.168.1.208

... 省略
# 在role:cluster-addon 中启用nfs-provisioner 安装
nfs_provisioner_install: "yes"			# 修改为yes
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "v4.0.1"
nfs_storage_class: "managed-nfs-storage"	
nfs_server: "192.168.31.244"			# 修改为实际nfs server地址
nfs_path: "/data/nfs"				# 修改为实际的nfs共享目录

$ dk ezctl setup ${集群名} 07 

# 执行成功后验证
$ kubectl get pod --all-namespaces |grep nfs-client
kube-system   nfs-client-provisioner-84ff87c669-ksw95      1/1     Running     0          21m

$ kubectl apply -f /etc/kubeasz/clusters/hello/yml/nfs-provisioner/test-pod.yaml

# 验证测试pod
kubectl get pod
NAME       READY   STATUS      RESTARTS   AGE
test-pod   0/1     Completed   0          6h36m

# 验证自动创建的pv 资源，
kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                STORAGECLASS          REASON   AGE
pvc-44d34a50-e00b-4f6c-8005-40f5cc54af18   2Mi        RWX            Delete           Bound    default/test-claim   managed-nfs-storage            6h36m

# 验证PVC已经绑定成功：STATUS字段为 Bound
kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS          AGE
test-claim   Bound    pvc-44d34a50-e00b-4f6c-8005-40f5cc54af18   2Mi        RWX            managed-nfs-storage   6h37m

.
└── default-test-claim-pvc-44d34a50-e00b-4f6c-8005-40f5cc54af18
    └── SUCCESS

... 省略
local_path_provisioner_install: "yes" # 修改为yes
# 设置默认本地存储路径
local_path_provisioner_dir: "/opt/local-path-provisioner"

$ dk ezctl setup ${集群名} 07

# 执行成功后验证
$ kubectl get pod --all-namespaces |grep provisioner

global_defs {
}

vrrp_track_process check-l4lb {
    process l4lb
    weight -60
    delay 3
}

vrrp_instance VI-01 {
    state MASTER
    priority 120
    unicast_src_ip {{ inventory_hostname }}
    unicast_peer {
{% for h in groups['ex_lb'] %}{% if h != inventory_hostname %}
        {{ h }}
{% endif %}{% endfor %}
    }
    dont_track_primary
    interface {{ LB_IF }}
    virtual_router_id {{ ROUTER_ID }}
    advert_int 3
    track_process {
        check-l4lb
    }
    virtual_ipaddress {
        {{ EX_APISERVER_VIP }}
    }
}

systemctl status l4lb 	# 检查进程状态
journalctl -u l4lb		# 检查进程日志是否有报错信息
systemctl status keepalived 	# 检查进程状态
journalctl -u keepalived	# 检查进程日志是否有报错信息

ip a				# 检查 master的 VIP地址是否存在

/* Detailed source-code truncated for AI context efficiency. */

~# ezctl new k8s-01
2021-01-19 10:48:23 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-01
2021-01-19 10:48:23 DEBUG set version of common plugins
2021-01-19 10:48:23 DEBUG cluster k8s-01: files successfully created.
2021-01-19 10:48:23 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-01/hosts'
2021-01-19 10:48:23 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-01/config.yml'

# 一键安装
ezctl setup k8s-01 all

# 或者分步安装，具体使用 ezctl help setup 查看分步安装帮助信息
# ezctl setup k8s-01 01
# ezctl setup k8s-01 02
# ezctl setup k8s-01 03
# ezctl setup k8s-01 04
...

+----------------+               +-----------------+
|ezctl 1.1.1.1   |               |cluster-aio:     |
+--+---+---+-----+               |                 |
   |   |   |                     |master 4.4.4.4   |
   |   |   +-------------------->+etcd   4.4.4.4   |
   |   |                         |node   4.4.4.4   |
   |   +--------------+          +-----------------+
   |                  |
   v                  v
+--+------------+ +---+----------------------------+
| cluster-1:    | | cluster-2:                     |
|               | |                                |
| master 2.2.2.1| | master 3.3.3.1/3.3.3.2         |
| etcd   2.2.2.2| | etcd   3.3.3.1/3.3.3.2/3.3.3.3 |
| node   2.2.2.3| | node   3.3.3.4/3.3.3.5/3.3.3.6 |
+---------------+ +--------------------------------+

# 登录amd64部署机
cd /etc/kubeasz; mv bin down /tmp/; scp -r /etc/kubeasz root@{_ip_arm64}:/etc/
# 复制完成后找回 bin 和 down 子目录
mv /tmp/bin /etc/kubeasz/; mv /tmp/down /etc/kubeasz/

cd /etc/kubeasz
# 下载基础部分
./ezdown -D
# 下载额外部分（如有）
./ezdown -X ...
# 运行部署容器
./ezdown -S
# 配置机器ssh免密码登录，集群所有节点都免密，包括待新增arm64节点
ssh-copy-id xx.xx.xx.xx
ssh-copy-id ...
# 复制kubeconfig
mkdir /root/.kube/; cp clusters/default/kubectl.kubeconfig /root/.kube/config

source ~/.bashrc
# 添加新节点 x.x.x.x
dk ezctl add-node default x.x.x.x

$ kubectl get node -owide
NAME           STATUS   ROLES    AGE    VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
k8s-x.x.x-19   Ready    master   5d8h   v1.33.1   x.x.x.19   <none>        Ubuntu 20.04.4 LTS   5.4.0-122-generic    containerd://2.1.1
k8s-x.x.x-90   Ready    node     5d8h   v1.33.1   x.x.x.90   <none>        Ubuntu 22.04.5 LTS   5.15.0-134-generic   containerd://2.1.1
k8s-x.x.x-91   Ready    node     5d8h   v1.33.1   x.x.x.91   <none>        Ubuntu 22.04.5 LTS   5.15.0-134-generic   containerd://2.1.1
k8s-x.x.x-93   Ready    node     79s    v1.33.1   x.x.x.93   <none>        Ubuntu 22.04.5 LTS   5.15.0-140-generic   containerd://2.1.1

$ kubectl describe node|grep beta.kubernetes.io/arch
Labels:             beta.kubernetes.io/arch=amd64
Labels:             beta.kubernetes.io/arch=amd64
Labels:             beta.kubernetes.io/arch=amd64
Labels:             beta.kubernetes.io/arch=arm64

stat -fc %T /sys/fs/cgroup/ 
cgroup2fs

# 删除集群
dk ezctl destroy xxxx

# 重启
reboot

# 启动后重新安装
dk ezctl setup xxxx all

apt update

apt install iptables -y

yum install iptables -y

zypper install iptables
ln -s /usr/sbin/iptables /sbin/iptables

export release=3.6.0
wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown

# 国内环境
./ezdown -D

./ezdown -X flannel
./ezdown -X prometheus
...

# 如果操作系统是ubuntu 22.04
./ezdown -P ubuntu_22

./ezdown -D
./ezdown -X flannel
./ezdown -X prometheus
...

./ezdown -S

sed -i 's/^INSTALL_SOURCE.*$/INSTALL_SOURCE: "offline"/g' /etc/kubeasz/example/config.yml

source ~/.bashrc
dk ezctl start-aio
# 或者执行 docker exec -it kubeasz ezctl start-aio

export release=3.6.7
wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown

# 国内环境
./ezdown -D
# 海外环境
#./ezdown -D -m standard

# 按需下载
./ezdown -X prometheus
...

./ezdown -P

./ezdown -S

docker exec -it kubeasz ezctl start-aio
# 如果安装失败，查看日志排除后，使用如下命令重新安装aio集群
# docker exec -it kubeasz ezctl setup default all

$ source ~/.bashrc
$ kubectl version         # 验证集群版本     
$ kubectl get node        # 验证节点就绪 (Ready) 状态
$ kubectl get pod -A      # 验证集群pod状态，默认已安装网络插件、coredns、metrics-server等
$ kubectl get svc -A      # 验证集群服务状态

"hosts": [
    "127.0.0.1",
{% if groups['ex_lb']|length > 0 %}
    "{{ hostvars[groups['ex_lb'][0]]['EX_APISERVER_VIP'] }}",
{% endif %}
{% for host in groups['kube_master'] %}
    "{{ host }}",
{% endfor %}
    "{{ CLUSTER_KUBERNETES_SVC_IP }}",
{% for host in MASTER_CERT_HOSTS %}
    "{{ host }}",
{% endfor %}
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],

# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
MASTER_CERT_HOSTS:
  - "10.1.1.1"
  - "k8s.test.io"
  #- "www.test.com"

$ ezctl setup ${集群名} 04 -t change_cert,restart_master

$ ezctl backup k8s-01
# 或者如下手动执行ansible命令
# ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml  playbooks/94.backup.yml

/etc/kubeasz/clusters/k8s-01/backup/
├── snapshot_202106201205.db
├── snapshot_202106211406.db
└── snapshot.db

$ ezctl restore k8s-01
# 或者如下手动执行ansible命令
# ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml  playbooks/95.restore.yml

$ ezctl clean k8s-01
# 或者如下手动执行ansible命令
# ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml  playbooks/99.clean.yml
$ ezctl setup k8s-01 01
$ ezctl setup k8s-01 02
$ ezctl setup k8s-01 03
$ ezctl setup k8s-01 04
$ ezctl setup k8s-01 05
...
$ ezctl restore k8s-01
# ansible-playbook -i clusters/k8s-01/hosts -e @clusters/k8s-01/config.yml  playbooks/95.restore.yml

docker exec -it kubeasz ezctl kca-renew xxx
# 或者使用 dk ezctl kca-renew xxx

ezctl help kcfg-adm
Usage: ezctl kcfg-adm <cluster> <args>
available <args>:
    -A     to add a client kubeconfig with a newly created user
    -D     to delete a client kubeconfig with the existed user
    -L     to list all of the users
    -e     to set expiry of the user certs in hours (ex. 24h, 8h, 240h)
    -t     to set a user-type (admin or view)
    -u     to set a user-name prefix

examples: ./ezctl kcfg-adm test-k8s -L
          ./ezctl kcfg-adm default -A -e 240h -t admin -u jack
          ./ezctl kcfg-adm default -D -u jim-202101162141

ezctl kcfg-adm k8s-01 -L
2021-01-24 16:32:43 INFO list-kcfg k8s-01
2021-01-24 16:32:43 INFO list-kcfg in cluster:k8s-01

USER                           TYPE            EXPIRY(+8h if in Asia/Shanghai)
---------------------------------------------------------------------------------

2021-01-24 16:32:43 INFO list-kcfg k8s-01 success

ezctl kcfg-adm k8s-01 -A -u user01 -e 24h -t view
2021-01-24 17:32:33 INFO add-kcfg k8s-01
2021-01-24 17:32:33 INFO add-kcfg in cluster:k8s-01 with user:user01-202101241732

PLAY [localhost] *****************************************************************************************************

...（此处省略输出） 

TASK [deploy : debug] ************************************************************************************************
ok: [localhost] => {
    "msg": "查看user01-202101241732自定义kubeconfig：/etc/kubeasz/clusters/k8s-01/ssl/users/user01-202101241732.kubeconfig"
}

PLAY RECAP ***********************************************************************************************************
localhost                  : ok=12   changed=10   unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

2021-01-24 17:32:41 INFO add-kcfg k8s-01 success

ezctl kcfg-adm k8s-01 -A -u user02 -e 240h -t admin
2021-01-24 18:38:47 INFO add-kcfg k8s-01
2021-01-24 18:38:47 INFO add-kcfg in cluster:k8s-01 with user:user02-202101241838

PLAY [localhost] *****************************************************************************************************

...（此处省略输出）

TASK [deploy : debug] ************************************************************************************************
ok: [localhost] => {
    "msg": "查看user02-202101241838自定义kubeconfig：/etc/kubeasz/clusters/k8s-01/ssl/users/user02-202101241838.kubeconfig"
}

PLAY RECAP ***********************************************************************************************************
localhost                  : ok=12   changed=9    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

2021-01-24 18:38:55 INFO add-kcfg k8s-01 success

ezctl kcfg-adm k8s-01 -L
2021-01-24 18:40:30 INFO list-kcfg k8s-01
2021-01-24 18:40:30 INFO list-kcfg in cluster:k8s-01

USER                           TYPE            EXPIRY(+8h if in Asia/Shanghai)
---------------------------------------------------------------------------------
user02-202101241838            cluster-admin   2021-02-03T10:34:00Z
user01-202101241732            view            2021-01-25T09:28:00Z

2021-01-24 18:40:31 INFO list-kcfg k8s-01 success

ezctl kcfg-adm k8s-01 -D -u user01-202101241732
2021-01-24 21:41:50 INFO del-kcfg k8s-01
2021-01-24 21:41:50 INFO del-kcfg in cluster:k8s-01 with user:user01-202101241732
clusterrolebinding.rbac.authorization.k8s.io "crb-user01-202101241732" deleted
2021-01-24 21:41:50 INFO del-kcfg k8s-01 success

ezctl kcfg-adm k8s-01 -L
2021-01-24 21:42:02 INFO list-kcfg k8s-01
2021-01-24 21:42:02 INFO list-kcfg in cluster:k8s-01

USER                           TYPE            EXPIRY(+8h if in Asia/Shanghai)
---------------------------------------------------------------------------------
user02-202101241838            cluster-admin   2021-02-03T10:34:00Z

2021-01-24 21:42:02 INFO list-kcfg k8s-01 success

# 编辑 roles/ex-lb/defaults/main.yml，配置如下变量
INGRESS_NODEPORT_LB: "yes"
INGRESS_TLS_NODEPORT_LB: "yes"

$ ezctl setup ${集群名} ex-lb

... 前文省略
listen kube_master
        bind 0.0.0.0:8443
        mode tcp
        option tcplog
        balance roundrobin
        server 192.168.1.1 192.168.1.1:6443 check inter 2000 fall 2 rise 2 weight 1
        server 192.168.1.2 192.168.1.2:6443 check inter 2000 fall 2 rise 2 weight 1

listen ingress-node
        bind 0.0.0.0:80
        mode tcp
        option tcplog
        balance roundrobin
        server 192.168.1.3 192.168.1.3:23456 check inter 2000 fall 2 rise 2 weight 1
        server 192.168.1.4 192.168.1.4:23456 check inter 2000 fall 2 rise 2 weight 1

listen ingress-node-tls
        bind 0.0.0.0:443
        mode tcp
        option tcplog
        balance roundrobin
        server 192.168.1.3 192.168.1.3:23457 check inter 2000 fall 2 rise 2 weight 1
        server 192.168.1.4 192.168.1.4:23457 check inter 2000 fall 2 rise 2 weight 1

# snapshot备份
$ ETCDCTL_API=3 etcdctl snapshot save backup.db
# 查看备份
$ ETCDCTL_API=3 etcdctl --write-out=table snapshot status backup.db

ezctl backup k8s-01

1 1 * * * /usr/bin/docker exec -i kubeasz ezctl backup k8s-01

{{ base_dir }}/clusters/k8s-01/backup

# ssh 免密码登录
$ ssh-copy-id 192.168.1.11

# 新增节点
$ ezctl add-etcd test-k8s 192.168.1.11

# 登录任意etcd节点验证etcd集群状态
$ export ETCDCTL_API=3 
$ etcdctl member list

# 验证所有etcd节点服务状态和日志
$ systemctl status etcd
$ journalctl -u etcd -f

# ssh 免密码登录
$ ssh-copy-id 192.168.1.11

# 新增节点
$ ezctl add-master test-k8s 192.168.1.11

# 同理，重复上面步骤再新增节点并自定义nodename
$ ezctl add-master test-k8s 192.168.1.12 k8s_nodename=master-03

# 在新节点master 服务状态
$ systemctl status kube-apiserver 
$ systemctl status kube-controller-manager
$ systemctl status kube-scheduler

# 查看新master的服务日志
$ journalctl -u kube-apiserver -f

# 查看集群节点，可以看到新 master节点 Ready, 并且禁止了POD 调度功能
$ kubectl get node
NAME           STATUS                     ROLES     AGE       VERSION
192.168.1.1    Ready,SchedulingDisabled   <none>    3h        v1.9.3
192.168.1.2    Ready,SchedulingDisabled   <none>    3h        v1.9.3
192.168.1.3    Ready                      <none>    3h        v1.9.3
192.168.1.4    Ready                      <none>    3h        v1.9.3
192.168.1.11   Ready,SchedulingDisabled   <none>    2h        v1.9.3	# 新增 master节点

$ ezctl del-master test-k8s 192.168.1.11  # 假设待删除节点 192.168.1.11

# ssh 免密码登录
$ ssh-copy-id 192.168.1.11

# 新增节点
$ ezctl add-node test-k8s 192.168.1.11

# 同理，重复上面步骤再新增节点并自定义nodename
$ ezctl add-node test-k8s 192.168.1.12 k8s_nodename=worker-03

# 验证新节点状态
$ kubectl get node

# 验证新节点的网络插件calico 或flannel 的Pod 状态
$ kubectl get pod -n kube-system

# 验证新建pod能否调度到新节点，略

# ssh 免密码登录（每个新节点都需要先做）
$ ssh-copy-id 192.168.1.21
$ ssh-copy-id 192.168.1.22
$ ssh-copy-id 192.168.1.23

# 一次性批量新增节点
$ ezctl add-nodes test-k8s 192.168.1.21 192.168.1.22 192.168.1.23

$ ssh-copy-id -p 10022 192.168.2.1
$ ezctl add-node test-k8s 192.168.2.1 ansible_ssh_port=10022

$ ezctl del-node test-k8s 192.168.1.11 # 假设待删除节点为 192.168.1.11

dk ezctl backup k8s-01

# snapshot备份
$ ETCDCTL_API=3 etcdctl snapshot save backup.db
# 查看备份
$ ETCDCTL_API=3 etcdctl --write-out=table snapshot status backup.db

export release=3.6.9
curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown
./ezdown -D -m ''

./ezdown -S
source ~/.bashrc
dk ezctl start-aio

ssh-copy-id ${worker1_ip}
dk ezctl add-node default ${worker1_ip}

ssh-copy-id ${worker2_ip}
dk ezctl add-node default ${worker2_ip}

$ sonobuoy run --plugin-env=e2e.E2E_EXTRA_ARGS="--ginkgo.v" --mode=certified-conformance

$ sonobuoy status

$ sonobuoy logs

$ outfile=$(sonobuoy retrieve)

mkdir ./results; tar xzf $outfile -C ./results

sonobuoy delete

$ git config --global user.email "[email protected]"
$ git config --global user.name "Your Name"

kubectl -n argocd patch secret argocd-secret -p '{"data": {"admin.password": null, "admin.passwordMtime": null}}'
kubectl -n argocd delete pods -l app.kubernetes.io/name=argocd-server

dex.config: |
    connectors:
    - type: oidc
      id: aliyun
      name: aliyun
      config:
        issuer: https://oauth.aliyun.com
        clientID: "406************"
        clientSecret: E8G***************************************************b6
        scopes:
        - profile
        - openid
        - aliuid
        getUserInfo: true
        userIDKey: uid
        userNameKey: uid
        claimMapping:
          preferred_username: name
          email: uid

data:
  policy.csv: |
    # 设置普通用户app-dev 只读权限
    p, role:app-dev, projects, get, *, allow
    p, role:app-dev, applications, get, *, allow
    p, role:app-dev, logs, get, *, allow
    p, role:app-dev, exec, create, */*, allow

    # 设置测试项目，所有权限
    p, role:app-dev, applications, *, test-project/*, allow
    
    # 阿里云子账号 ID：2***********84
    g, "2***********84", role:admin
    g, "2***********27", role:app-dev

  policy.default: role:''
  scopes: '[name]'

data:
  # 设置argocd 资源标记方式，使用annotation，禁用labelKey
  # application.instanceLabelKey: argocd.argoproj.io/instance
  application.resourceTrackingMethod: annotation

data:
  #application.namespaces: app-team-one, app-team-two
  application.namespaces: '*'
  applicationsetcontroller.allowed.scm.providers: '*'
  applicationsetcontroller.namespaces: '*'

# 修改argocd-cm configmap，重启argocd-application-controller
data:
  resource.customizations: |
    networking.k8s.io/Ingress:
      health.lua: |
        hs = {}
        hs.status = "Healthy"
        hs.message = "Skip health check for Ingress"
        return hs

$ ansible -i clusters/${cluster_name}/hosts all -m shell -a 'timedatectl'
192.168.1.1 | SUCCESS | rc=0 >>
      Local time: Sat 2019-01-26 11:51:51 HKT
  Universal time: Sat 2019-01-26 03:51:51 UTC
        RTC time: Sat 2019-01-26 03:51:52
       Time zone: Asia/Hong_Kong (HKT, +0800)
 Network time on: yes
NTP synchronized: yes
 RTC in local TZ: no

192.168.1.4 | SUCCESS | rc=0 >>
      Local time: Sat 2019-01-26 11:51:51 HKT
  Universal time: Sat 2019-01-26 03:51:51 UTC
        RTC time: Sat 2019-01-26 03:51:52
       Time zone: Asia/Hong_Kong (HKT, +0800)
 Network time on: yes
NTP synchronized: yes
 RTC in local TZ: no

192.168.1.2 | SUCCESS | rc=0 >>
      Local time: Sat 2019-01-26 11:51:51 HKT
  Universal time: Sat 2019-01-26 03:51:51 UTC
        RTC time: Sat 2019-01-26 03:51:52
       Time zone: Asia/Hong_Kong (HKT, +0800)
 Network time on: yes
NTP synchronized: yes
 RTC in local TZ: no

192.168.1.3 | SUCCESS | rc=0 >>
      Local time: Sat 2019-01-26 11:51:51 HKT
  Universal time: Sat 2019-01-26 03:51:51 UTC
        RTC time: Sat 2019-01-26 03:51:52
       Time zone: Asia/Hong_Kong (HKT, +0800)
 Network time on: yes
NTP synchronized: yes
 RTC in local TZ: no

# 部署dashboard 主yaml配置文件
$ kubectl create -f /etc/kubeasz/manifests/dashboard/1.6.3/kubernetes-dashboard.yaml
# 部署基本密码认证配置[可选]，密码文件位于 /etc/kubernetes/ssl/basic-auth.csv
$ kubectl create -f /etc/kubeasz/manifests/dashboard/1.6.3/ui-admin-rbac.yaml
$ kubectl create -f /etc/kubeasz/manifests/dashboard/1.6.3/ui-read-rbac.yaml

# 查看pod 运行状态
kubectl get pod -n kube-system | grep dashboard
kubernetes-dashboard-86bd8778bf-w4974      1/1       Running   0          12h
# 查看dashboard service
kubectl get svc -n kube-system|grep dashboard
kubernetes-dashboard   NodePort    10.68.7.67      <none>        80:5452/TCP	12h
# 查看集群服务
kubectl cluster-info|grep dashboard
kubernetes-dashboard is running at https://192.168.1.10:6443/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy
# 查看pod 运行日志，关注有没有错误
kubectl logs kubernetes-dashboard-86bd8778bf-w4974 -n kube-system

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: ui-admin
rules:
- apiGroups:
  - ""
  resources:
  - services
  - services/proxy
  verbs:
  - '*'

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ui-admin-binding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ui-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: admin

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: ui-read
rules:
- apiGroups:
  - ""
  resources:
  - services
  - services/proxy
  verbs:
  - get
  - list
  - watch

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ui-read-binding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ui-read
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: readonly

# ezctl 集成部署组件，xxxx 代表集群部署名
# dashboard 部署文件位于 /etc/kubeasz/clusters/xxxx/yml/dashboard/ 目录
./ezctl setup xxxx 07

# 查看pod 运行状态
kubectl get pod -n kube-system | grep dashboard
dashboard-metrics-scraper-856586f554-l6bf4   1/1     Running   0          35m
kubernetes-dashboard-698d4c759b-67gzg        1/1     Running   0          35m

# 查看dashboard service
kubectl get svc -n kube-system|grep dashboard
kubernetes-dashboard   NodePort    10.68.219.38   <none>        443:24108/TCP                   53s

# 查看pod 运行日志
kubectl logs -n kube-system kubernetes-dashboard-698d4c759b-67gzg

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets admin-user

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets dashboard-read-user

apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: LS0tLS1CRUdxxxxxxxxxxxxxx
    server: https://192.168.1.2:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: admin
  name: kubernetes
current-context: kubernetes
kind: Config
preferences: {}
users:
- name: admin
  user:
    client-certificate-data: LS0tLS1CRUdJTiBDRxxxxxxxxxxx
    client-key-data: LS0tLS1CRUdJTxxxxxxxxxxxxxx
    token: eyJhbGcixxxxxxxxxxxxxxxx

# 1. 修改 clusters/default/config.yml 文件，设置 dashboard_install: "yes"

# 2. 下载dashboard 需要的镜像
./ezdown -X dashboard

# 3. 执行安装，配置文件位于 clusters/default/yml/dashboard/ 目录
dk ezctl setup default 07

# 查看pod 运行状态
kubectl get pod -n kube-system |grep kubernetes-dashboard
kubernetes-dashboard-api-6d77cb7964-4tklq               1/1     Running   0          17h
kubernetes-dashboard-auth-5fbd64f659-f9dst              1/1     Running   0          17h
kubernetes-dashboard-kong-6dcdbf5dfd-829h4              1/1     Running   0          17h
kubernetes-dashboard-metrics-scraper-7757c48476-4lcrq   1/1     Running   0          17h
kubernetes-dashboard-web-5f9f47979-7khrk                1/1     Running   0          17h

# 查看service
kubectl get svc -n kube-system |grep kong
kubernetes-dashboard-kong-proxy        NodePort    10.68.148.170   <none>   443:31544/TCP  17h

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets admin-user

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets dashboard-read-user

ezdown -D
ezdown -R

#clusters/xxx/hosts 中修改如下，配置harbor组下机器，设置NEW_INSTALL=true
...
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
192.168.1.8 NEW_INSTALL=true
...

#clusters/xxx/config.yml 中修改如下，按需修改HARBOR_DOMAIN/HARBOR_TLS_PORT 等配置项，举例如下
############################
# role:harbor
############################
# harbor version，完整版本号
HARBOR_VER: "v2.10.2"
HARBOR_DOMAIN: "harbor.yourdomain.com"
HARBOR_PATH: /var/data
HARBOR_TLS_PORT: 8443
HARBOR_REGISTRY: "{{ HARBOR_DOMAIN }}:{{ HARBOR_TLS_PORT }}"

# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: true

# install component
HARBOR_WITH_TRIVY: false

docker login harbor.test.com
Username: 
Password:
Login Succeeded
docker tag busybox:latest harbor.test.com/library/busybox:latest
docker push harbor.test.com/library/busybox:latest
The push refers to a repository [harbor.test.com/library/busybox]
0271b8eebde3: Pushed 
latest: digest: sha256:91ef6c1c52b166be02645b8efee30d1ee65362024f7da41c404681561734c465 size: 527

apiVersion: v1
kind: Pod
metadata:
  name: test-busybox
spec:
  containers:
  - name: test-busybox
    image: harbor.test.com/xxx/busybox:latest
    imagePullPolicy: Always

apiVersion: v1
kind: Pod
metadata:
  name: test-busybox
spec:
  containers:
  - name: test-busybox
    image: harbor.test.com/xxx/busybox:latest
    imagePullPolicy: Always
  imagePullSecrets:
  - name: harborkey1

//harborkey1.yaml
apiVersion: v1
kind: Secret
metadata:
  name: harborkey1
  namespace: default
data:
    .dockerconfigjson: {base64 -w 0 ~/.docker/config.json}
type: kubernetes.io/dockerconfigjson

# 停止 harbor
 docker-compose down -v
# 修改配置
 vim harbor.yml
# 执行./prepare已更新配置到docker-compose.yml文件
 ./prepare
# 启动 harbor
 docker-compose up -d

# 1. 修改 clusters/default/config.yml 文件，设置 headlamp_install: "yes"

# 2. 下载需要的镜像
./ezdown -X headlamp

# 3. 执行安装，配置文件位于 clusters/default/yml/headlamp/ 目录
dk ezctl setup default 07

# 查看 Pod 运行状态
kubectl get pod -n kube-system -l app.kubernetes.io/name=headlamp

# 查看 Service
kubectl get svc -n kube-system headlamp

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets admin-user

# 获取 Bearer Token，找到输出中 ‘token:’ 开头的后面部分
$ kubectl describe -n kube-system secrets dashboard-read-user

wget https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz
mv ./linux-amd64/helm /usr/bin

helm repo add stable https://kubernetes-charts.storage.googleapis.com/

helm repo add stable http://mirror.azure.cn/kubernetes/charts

mkdir -p /opt/charts/redis-cluster
cd /opt/charts/redis-cluster

helm repo update
helm pull stable/redis-ha

tar zxvf redis-ha-*.tgz
cp redis-ha/values.yaml .

cat > start.sh << EOF
#!/bin/sh
set -x

ROOT=$(cd `dirname $0`; pwd)
cd $ROOT

helm install redis \
	--create-namespace \
	--namespace dependency \
	-f ./values.yaml \
	./redis-ha
EOF

tree .
.
├── redis-ha		# redis-ha 原始charts目录
├── start.sh		# 启动命名脚本
└── values.yaml		# 个性化参数配置

#举例values.yaml 配置如下，没有启用PV
#cat values.yaml
image:
  repository: redis
  tag: 5.0.6-alpine

replicas: 2

## Redis specific configuration options
redis:
  port: 6379
  masterGroupName: "mymaster"       # must match ^[\\w-\\.]+$) and can be templated
  config:
    ## For all available options see http://download.redis.io/redis-stable/redis.conf
    min-replicas-to-write: 1
    min-replicas-max-lag: 5   # Value in seconds
    maxmemory: "4g"       # Max memory to use for each redis instance. Default is unlimited.
    maxmemory-policy: "allkeys-lru"  # Max memory policy to use for each redis instance. Default is volatile-lru.
    repl-diskless-sync: "yes"
    rdbcompression: "yes"
    rdbchecksum: "yes"

  resources:
    requests:
      memory: 200Mi
      cpu: 100m
    limits:
      memory: 4000Mi

## Sentinel specific configuration options
sentinel:
  port: 26379
  quorum: 1

  resources:
    requests:
      memory: 200Mi
      cpu: 100m
    limits:
      memory: 200Mi

hardAntiAffinity: true

## Configures redis with AUTH (requirepass & masterauth conf params)
auth: false

persistentVolume:
  enabled: false

hostPath:
  path: "/data/mcs-redis/{{ .Release.Name }}"

bash ./start.sh

helm ls -A
NAME 	NAMESPACE 	REVISION	UPDATED                                	STATUS  	CHART         	APP VERSION
redis	dependency	1       	2020-05-28 20:57:31.166002853 +0800 CST	deployed	redis-ha-4.4.4	5.0.6

# 查看k8s上资源
kubectl get pod,svc -n dependency
NAME                          READY   STATUS    RESTARTS   AGE
pod/redis-redis-ha-server-0   2/2     Running   0          119s
pod/redis-redis-ha-server-1   2/2     Running   0          104s

NAME                                TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)              AGE
service/redis-redis-ha              ClusterIP   None          <none>        6379/TCP,26379/TCP   119s
service/redis-redis-ha-announce-0   ClusterIP   10.68.41.65   <none>        6379/TCP,26379/TCP   119s
service/redis-redis-ha-announce-1   ClusterIP   10.68.64.49   <none>        6379/TCP,26379/TCP   119s

# 创建deploy和service
$ kubectl run php-apache --image=pilchard/hpa-example --requests=cpu=200m --expose --port=80

# 创建autoscaler
$ kubectl autoscale deploy php-apache --cpu-percent=50 --min=1 --max=10

# 等待3~5分钟查看hpa状态
$ kubectl get hpa php-apache
NAME         REFERENCE               TARGETS    MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   0% / 50%   1         10        1          3m

# 增加负载
$ kubectl run --rm -it load-generator --image=busybox /bin/sh
Hit enter for command prompt
$ while true; do wget -q -O- http://php-apache; done;

# 等待约5分钟查看hpa显示负载增加，且副本数目增加为4
$ kubectl get hpa php-apache
NAME         REFERENCE               TARGETS      MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   430% / 50%   1         10        4          4m

# 注意k8s为了避免频繁增删pod，对副本的增加速度有限制
# 实验过程可以看到副本数目从1到4到8到10，大概都需要4~5分钟的缓冲期
$ kubectl get hpa php-apache
NAME         REFERENCE               TARGETS     MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   86% / 50%   1         10        8          9m
$ kubectl get hpa php-apache
NAME         REFERENCE               TARGETS     MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   52% / 50%   1         10        10         12m

# 清除负载，CTRL+C 结束上述循环程序，稍后副本数目变回1
$ kubectl get hpa php-apache
NAME         REFERENCE               TARGETS    MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   0% / 50%   1         10        1          17m

kubectl create -f /etc/kubeasz/manifests/ingress/traefik/traefik-ingress.yaml

# kubectl get deploy -n kube-system traefik-ingress-controller
NAME                         DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
traefik-ingress-controller   1         1         1            1           4m

# kubectl get svc -n kube-system traefik-ingress-service
NAME                      TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)                       AGE
traefik-ingress-service   NodePort   10.68.69.170   <none>        80:23456/TCP,8080:34815/TCP   4m

kubectl run test-hello --image=nginx:alpine --expose --port=80
##
# kubectl get deploy test-hello
NAME         DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
test-hello   1         1         1            1           56s
# kubectl get svc test-hello
NAME         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
test-hello   ClusterIP   10.68.124.115   <none>        80/TCP    1m

# test-hello.ing.yaml内容
apiVersion: networking.k8s.io/v1beta1 
kind: Ingress
metadata:
  name: test-hello
spec:
  rules:
  - host: hello.test.com
    http:
      paths:
      - path: /
        backend:
          serviceName: test-hello
          servicePort: 80

# traefik-ui.ing.yaml内容
---
apiVersion: networking.k8s.io/v1beta1 
kind: Ingress
metadata:
  name: traefik-web-ui
  namespace: kube-system
spec:
  rules:
  - host: traefik-ui.test.com
    http:
      paths:
      - path: /
        backend:
          serviceName: traefik-ingress-service
          servicePort: 8080

192.168.1.1	hello.test.com
192.168.1.1	traefik-ui.test.com

# 修改traefik-ingress 使用 LoadBalancer服务
$ sed -i 's/NodePort$/LoadBalancer/g' /etc/kubeasz/manifests/ingress/traefik/traefik-ingress.yaml
# 创建traefik-ingress
$ kubectl apply -f /etc/kubeasz/manifests/ingress/traefik/traefik-ingress.yaml
# 验证
$ kubectl get svc --all-namespaces |grep traefik
kube-system   traefik-ingress-service   LoadBalancer   10.68.163.243   192.168.1.241   80:23456/TCP,8080:37088/TCP   1m

192.168.1.241     hello.test.com
192.168.1.241     traefik-ui.test.com

192.168.1.10     hello.test.com
192.168.1.10    traefik-ui.test.com