Ansible-K8S 二进制安装 - YIDADA-SRE

link管理
链接快照平台
输入网页链接，自动生成快照
标签化管理网页链接
相关文章推荐
帅气的生姜 · 4.深入k8s：持久卷PV、PVC及其源码分 ...· 2 周前 ·
飘逸的饼干 · 从动态获取div高度的问题展开来看 - ...· 2 月前 ·
瘦瘦的跑步鞋 · 科技部调研青岛海洋科学与技术国家实验室建设情况· 1 年前 ·
飞奔的铁板烧 · 机器学习&深度学习常用shell命令小结-C ...· 2 年前 ·
风流倜傥的大蒜 · 京都市營巴士・地下鐵導遊指南：地下鐵一日乘車券· 2 年前 ·
留胡子的柑橘 · protected override ...· 2 年前 ·
cat > /etc/apt/sources.list <<'EOF'
deb http://mirrors.aliyun.com/ubuntu/ focal main restricted   #优先检索focal的main和restricted组件。
deb http://mirrors.aliyun.com/ubuntu/ focal-updates main restricted
deb http://mirrors.aliyun.com/ubuntu/ focal universe
deb http://mirrors.aliyun.com/ubuntu/ focal-updates universe
deb http://mirrors.aliyun.com/ubuntu/ focal multiverse
deb http://mirrors.aliyun.com/ubuntu/ focal-updates multiverse
deb http://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu focal-security main restricted
deb http://mirrors.aliyun.com/ubuntu focal-security universe
deb http://mirrors.aliyun.com/ubuntu focal-security multiverse
apt-get update
apt-get update
系统主机名
hostnamectl set-hostname xxxx
IP地址配置
vim /etc/netplan/00-installer-config.yaml
系统参数优化
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
sudo modprobe overlay
sudo modprobe br_netfilter
# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
# 应用 sysctl 参数而不重新启动
sudo sysctl --system
关闭交换分区
swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab
关闭selinux
setenforce 0
ufw disable
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
date -R
cat > /etc/hosts <<'EOF'
192.168.1.70 k8s-master1
192.168.1.71 k8s-master2
192.168.1.72 k8s-ha1 harbor.nbrhce.com
192.168.1.73 k8s-etcd1
192.168.1.74 k8s-etcd2
192.168.1.75 k8s-etcd3
192.168.1.76 k8s-node1
2. 负载均衡部署
2.1 keepalived部署
root@k8s-ha1:~#  apt install keepalived -y
root@k8s-ha1:~#  find / -name keepalived*
#找到这个文件更改配置即可
/usr/share/doc/keepalived/samples/keepalived.conf.vrrp
#参考模板
root@k8s-ha1:~#  cp /usr/share/doc/keepalived/samples/keepalived.conf.vrrp /etc/keepalived/keepalived.conf
root@k8s-ha1:~# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 3
    authentication {
        auth_type PASS
        auth_pass 123abc
    virtual_ipaddress {
        192.168.1.72 dev eth0 label eth0:1
root@localhost:~# cat /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 50
    advert_int 3
    authentication {
        auth_type PASS
        auth_pass 123abc
    virtual_ipaddress {
        192.168.1.73 dev eth0 label eth0:1
systemctl start keepalived.service 
systemctl enable keepalived.service 
#检查是否出现VIP
root@k8s-ha1:~# ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.1.73  netmask 255.255.255.0  broadcast 192.168.1.255
        inet6 fe80::20c:29ff:feab:bfb7  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:ab:bf:b7  txqueuelen 1000  (Ethernet)
        RX packets 492885  bytes 250991107 (250.9 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 387918  bytes 23556907 (23.5 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.1.72  netmask 255.255.255.255  broadcast 0.0.0.0
        ether 00:0c:29:ab:bf:b7  txqueuelen 1000  (Ethernet)
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 182  bytes 15942 (15.9 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 182  bytes 15942 (15.9 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
2.2 haproxy部署
root@k8s-ha1:~# apt install -y haproxy
#主备服务器配置一样，当ha1服务器宕机了 VIP会跑到ha2上面 但是有瑕疵
root@k8s-ha1:~# vim /etc/haproxy/haproxy.cfg
listen k8s_api_nodes_6443
#这里绑定的是VIP地址加端口
        bind 192.168.1.100:6443
        mode tcp
        server 192.168.1.70 192.168.1.70:6443 check inter 2000 fall 3 rise 5
        server 192.168.1.71 192.168.1.71:6443 check inter 2000 fall 3 rise 5
systemctl restart haproxy.service
systemctl enable haproxy.service 
已经监听了6443端口
root@k8s-ha1:~# ss -lntup
Netid    State     Recv-Q    Send-Q       Local Address:Port       Peer Address:Port   Process                                       
udp      UNCONN    0         0            127.0.0.53%lo:53              0.0.0.0:*       users:(("systemd-resolve",pid=790,fd=12))    
tcp      LISTEN    0         262124        192.168.1.69:6443            0.0.0.0:*       users:(("haproxy",pid=6349,fd=7))            
tcp      LISTEN    0         4096         127.0.0.53%lo:53              0.0.0.0:*       users:(("systemd-resolve",pid=790,fd=13))    
tcp      LISTEN    0         128                0.0.0.0:22              0.0.0.0:*       users:(("sshd",pid=854,fd=3))                
tcp      LISTEN    0         128              127.0.0.1:6010            0.0.0.0:*       users:(("sshd",pid=1151,fd=11))              
tcp      LISTEN    0         128                   [::]:22                 [::]:*       users:(("sshd",pid=854,fd=4))                
tcp      LISTEN    0         128                  [::1]:6010               [::]:*       users:(("sshd",pid=1151,fd=10)) 
#有个缺点这里少一个脚本 来监控haproxy 功能是只要haproxy宕机了就不要让他运行了
3. harbor部署
由于我机器资源不够harbor服务与ansible在一台机器上
kubernetes master节点和node节点使用containerd 可以提前安装或使用部署工具安装，在harbor节点安装docker，后续需要部署harbor




    

3.1  Harbor部署
#我这里直接使用脚本部署了或者自行安装docker大于1.19版本
root@k8s-harbor1:~# tar xf docker-list.tar.gz
root@k8s-harbor1:~# bash docker-install.sh
root@k8s-harbor1:~# docker version
Client: Docker Engine - Community
 Version:           19.03.15
 API version:       1.40
 Go version:        go1.13.15
 Git commit:        99e3ed8
 Built:             Sat Jan 30 03:11:43 2021
 OS/Arch:           linux/amd64
 Experimental:      false
Server: Docker Engine - Community
 Engine:
  Version:          19.03.15
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.13.15
  Git commit:       99e3ed8
  Built:            Sat Jan 30 03:18:13 2021
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          v1.3.9
  GitCommit:        ea765aba0d05254012b0b9e595e995c09186427f
 runc:
  Version:          1.0.0-rc10
  GitCommit:        dc9208a3303feef5b3839f4323d9beb36df0a9dd
 docker-init:
  Version:          0.18.0
  GitCommit:        fec3683
3.2 Harbor配置https自签证书
业务镜像将统一上传到Harbor的服务器并实现镜像分发，不用通过互联网在线下载公网镜像，提高镜像分发效率及数据安全性
root@k8s-harbor1:~# mkdir /apps/certs -p 
#生成CA证书私钥
root@k8s-harbor1:/apps/certs# openssl genrsa -out ca.key 4096
#根据上面生成的CA证书私钥，再来生成CA证书 ca.crt 生成 CA 证书
root@k8s-harbor1:/apps/certs# openssl req -x509 -new -nodes -sha512 -days 3650 \
            -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=harbor.nbrhce.com" \
            -key ca.key \
            -out ca.crt
#转成一个客户端证书
root@k8s-harbor1:/apps/certs#  openssl genrsa -out harbor.nbrhce.com.key 4096
#生成证书签名请求（CSR）yourdomain.com.csr。生成证书签名请求 （CSR）
root@k8s-harbor1:/apps/certs# openssl req -sha512 -new \
            -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=harbor.nbrhce.com" \
            -key harbor.nbrhce.com.key \
            -out harbor.nbrhce.com.csr
#生成一个 x509 v3扩展文件。生成 x509 v3 扩展文件
root@k8s-harbor1:/apps/certs# cat > v3.ext << 'EOF'
            authorityKeyIdentifier=keyid,issuer
            basicConstraints=CA:FALSE
            keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
            extendedKeyUsage = serverAuth
            subjectAltName = @alt_names
#写一个对应的域名即可 下面两个随便添
            [alt_names]
            DNS.1=harbor.nbrhce.com
            DNS.2=harbor.nbrhce.net
            DNS.3=harbor.nbrhce.local
# 使用该文件为您的Harbor主机生成证书 harbor.nbrhce.com.cert使用该文件为您的 Harbor 主机生成证书 改成对应域名为.cert结尾即可
root@k8s-harbor1:/apps/certs# openssl x509 -req -sha512 -days 3650 \
            -extfile v3.ext \
            -CA ca.crt -CAkey  ca.key -CAcreateserial \
            -in harbor.nbrhce.com.csr \
            -out harbor.nbrhce.com.crt
#在生成一个客户端证书留着用
root@k8s-harbor1:/apps/certs# openssl x509 -inform PEM -in harbor.nbrhce.com.key -out harbor.nbrhce.com.cert
3.3 配置harbor证书信任
./install.sh  --with-trivy --with-chartmuseum 
#配置证书信任 必须在这个路径下创建一个目录对应你所使用的域名 否则对应的节点与容器识别不到你的域名
root@k8s-harbor1:/apps/certs#  mkdir /etc/docker/certs.d/harbor.nbrhce.com -p
#把生成的证书放在这个路径下 我就全部都拷贝过来了 或者拷贝一个cert也行
root@k8s-ha-deploy:/apps/certs# cp harbor.nbrhce.com.crt /etc/docker/certs.d/harbor.nbrhce.com/
 root@k8s-ha-deploy:/apps/certs# cp ca.crt /etc/docker/certs.d/harbor.nbrhce.com/
root@k8s-ha-deploy:/apps/certs# cp ca.key /etc/docker/certs.d/harbor.nbrhce.com/
#在这个文件中启动服务这块 加上对应访问的域名 与加上harbor主机IP地址与5000端口 这个端口是容器内部代理，你是在宿主机上是看不到的
root@k8s-ha-deploy:/apps/certs# vim /lib/systemd/system/docker.service
 ExecStart=/usr/bin/dockerd -H fd://  --insecure-registry=192.168.1.72:5000 --insecure-registry=harbor.nbrhce.com  --containerd=/run/containerd/containerd.sock
更改完必要忘记重启&&重启完docker你会发现harbor也会宕机 你也需要重新启动一下
systemctl daemon-reload
systemctl restart docker
docker-compose restart  或者 docker-compsoe up -d 
#测试 拉取镜像并上传到私有库 先登录 这个用户是harbor内部用户 你先需要去harbor里创建一个用户并给他授权成管理员 这样这个用户就有权限上传镜像了
root@k8s-ha-deploy:/apps/harbor# docker login harbor.nbrhce.com
Username:
password:
root@k8s-ha-deploy:/apps/harbor# docker pull apline
root@k8s-ha-deploy:/apps/harbor# docker tag alpine:latest harbor.nbrhce.com/baseimages/alpine:latest
root@k8s-ha-deploy:/apps/harbor# docker push harbor.nbrhce.com/baseimages/alpine:latest
有这个小锁头就代表配置成功了

for node in ${IP};do
 sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
 echo "${node} 密钥copy完成"
 ssh ${node} ln -sv /usr/bin/python3 /usr/bin/python
 echo "${node} /usr/bin/python3 软连接创建完成"
#秘钥分发完要测试一下python的软链接是否生成 
5. 下载kubeasz开源项目及组件
会有docker环境 是因为先用docker拉去镜像拷贝到容器里面在删除下载的镜像
hub.docker.io kubeasz-k8s-bin
5.1 下载开源项目
root@k8s-deploy:~# apt install git -y
root@k8s-deploy:~# export release=3.3.1
root@k8s-deploy:~# wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
root@k8s-deploy:~# vim ezdown #自定义下载组件版本-这次不用改
root@k8s-deploy:~# chmod a+x ./ezdown
root@k8s-deploy:~# ./ezdown -D
root@k8s-deploy:~# ll /etc/kubeasz/
total 136
drwxrwxr-x 12 root root  4096 Nov 17 13:22 ./
drwxr-xr-x 95 root root  4096 Nov 17 13:22 ../
-rw-rw-r--  1 root root 20304 Jul  3 20:37 ansible.cfg
drwxr-xr-x  3 root root  4096 Nov 17 13:22 bin/
drwxrwxr-x  8 root root  4096 Jul  3 20:51 docs/
drwxr-xr-x  2 root root  4096 Nov 17 13:30 down/
drwxrwxr-x  2 root root  4096 Jul  3 20:51 example/
-rwxrwxr-x  1 root root 25012 Jul  3 20:37 ezctl*
-rwxrwxr-x  1 root root 25266 Jul  3 20:37 ezdown*
drwxrwxr-x  3 root root  4096 Jul  3 20:51 .github/
-rw-rw-r--  1 root root   301 Jul  3 20:37 .gitignore
drwxrwxr-x 10 root root  4096 Jul  3 20:51 manifests/
drwxrwxr-x  2 root root  4096 Jul  3 20:51 pics/
drwxrwxr-x  2 root root  4096 Jul  3 20:51 playbooks/
-rw-rw-r--  1 root root  5058 Jul  3 20:37 README.md
drwxrwxr-x 22 root root  4096 Jul  3 20:51 roles/
drwxrwxr-x  2 root root  4096 Jul  3 20:51 tools/
5.2 创建集群
#创建一个k8s集群 这个过程就是创建一个目录 目录下有hosts config
root@k8s-deploy:/etc/kubeasz# ./ezctl new k8s-cluster1
2022-11-17 13:38:04 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-cluster1
2022-11-17 13:38:04 DEBUG set versions
2022-11-17 13:38:04 DEBUG cluster k8s-cluster1: files successfully created.
2022-11-17 13:38:04 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-cluster1/hosts'
2022-11-17 13:38:04 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-cluster1/config.yml'
5.3 更改hosts文件
#这个文件就是加上你的机器节点IP地址
root@k8s-ha1:/etc/kubeasz/clusters/k8s-cluster1# cat hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.1.76
192.168.1.77
192.168.1.78
# master node(s)
[kube_master]
192.168.1.70
192.168.1.71
# work node(s)
[kube_node]
192.168.1.79
# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
#192.168.1.8 NEW_INSTALL=false
# [optional] loadbalance for accessing k8s from outside
[ex_lb]
#192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
# [optional] ntp server for the cluster
[chrony]
#192.168.1.1
[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"
# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"
#=================================这里更改了子网IP变大一点=============================#
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"
#=================================这里更改了子网IP变大一点=============================#
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"
# NodePort Range
NODE_PORT_RANGE="30000-32767"
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local"
# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
#=================================这里更改命令的存放位置===============================#
bin_dir="/usr/local/bin"
# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"
# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/k8s-cluster1"
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"
root@k8s-ha1:/etc/kubeasz/clusters/k8s-cluster1# 
5.4 config.yaml
#更改初始化容器镜像
#原本是这个为了更快打个标签上传到私有仓库 记得把文件内容更改一下 这个是初始化容器的镜像这个拉取不下来就不可能初始化成功
easzlab.io.local:5000/easzlab/pause:3.7
harbor.nbrhce.com/baseimages/pause:3.7
root@k8s-ha1:/etc/kubeasz/clusters/k8s-cluster1# cat config.yml
############################
# prepare
############################
# 可选离线安装系统软件包 (offline|online)
INSTALL_SOURCE: "online"
# 可选进行系统安全加固 github.com/dev-sec/ansible-collection-hardening
OS_HARDEN: false
############################
# role:deploy
############################
# default: ca will expire in 100 years
# default: certs issued by the ca will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"
# kubeconfig 配置参数
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"
# k8s version
K8S_VER: "1.24.2"
############################
# role:etcd
############################
# 设置不同的wal目录，可以避免磁盘io竞争，提高性能
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""
############################
# role:runtime [containerd,docker]
############################
# ------------------------------------------- containerd
# [.]启用容器仓库镜像
ENABLE_MIRROR_REGISTRY: false
# [containerd]基础容器镜像
#=================================这里更改了镜像======================================#
SANDBOX_IMAGE: "harbor.nbrhce.com/baseimages/pause:3.7"
# [containerd]容器持久化存储目录
CONTAINERD_STORAGE_DIR: "/var/lib/containerd"
# ------------------------------------------- docker
# [docker]容器存储目录
DOCKER_STORAGE_DIR: "/var/lib/docker"
# [docker]开启Restful API
ENABLE_REMOTE_API: false
# [docker]信任的HTTP仓库
#==============================这里添加了自己harbor仓库地址============================#
INSECURE_REG: '["http://easzlab.io.local:5000","harbor.nbrhce.com"]'
############################
# role:kube-master
############################
# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
#=================================这里更改IP为VIP地址==域名为随便起一个=================#
MASTER_CERT_HOSTS:
  - "192.168.1.72"
  - "api.myserver.com"
  #- "www.test.com"
# node 节点上 pod 网段掩码长度（决定每个节点最多能分配的pod ip地址）
# 如果flannel 使用 --kube-subnet-mgr 参数，那么它将读取该设置为每个节点分配pod网段
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 23
############################
# role:kube-node
############################
# Kubelet 根目录
KUBELET_ROOT_DIR: "/var/lib/kubelet"
# node节点最大pod 数
#=================================这里更改为500======================================#
MAX_PODS: 500
# 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
# 数值设置详见templates/kubelet-config.yaml.j2
KUBE_RESERVED_ENABLED: "no"
# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
# 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
# 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
# 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
SYS_RESERVED_ENABLED: "no"
############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel]设置flannel 后端"host-gw","vxlan"等
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false
# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "v0.15.1"
flanneld_image: "easzlab.io.local:5000/easzlab/flannel:{{ flannelVer }}"
# ------------------------------------------- calico
# [calico]设置 CALICO_IPV4POOL_IPIP=“off”,可以提高网络性能，条件限制详见 docs/setup/calico.md
CALICO_IPV4POOL_IPIP: "Always"
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
# [calico]设置calico 网络 backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"
# [calico]设置calico 是否使用route reflectors
# 如果集群规模超过50个节点，建议启用该特性
CALICO_RR_ENABLED: false
# CALICO_RR_NODES 配置route reflectors的节点，如果未设置默认使用集群master节点 
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []
# [calico]更新支持calico 版本: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "v3.19.4"
# [calico]calico 主版本
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
# ------------------------------------------- cilium
# [cilium]镜像版本
cilium_ver: "1.11.6"
cilium_connectivity_check: true
cilium_hubble_enabled: false
cilium_hubble_ui_enabled: false
# ------------------------------------------- kube-ovn
# [kube-ovn]选择 OVN DB and OVN Control Plane 节点，默认为第一个master节点
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"
# [kube-ovn]离线镜像tar包
kube_ovn_ver: "v1.5.3"
# ------------------------------------------- kube-router
# [kube-router]公有云上存在限制，一般需要始终开启 ipinip；自有环境可以设置为 "subnet"
OVERLAY_TYPE: "full"
# [kube-router]NetworkPolicy 支持开关
FIREWALL_ENABLE: true
# [kube-router]kube-router 镜像版本
kube_router_ver: "v0.3.1"
busybox_ver: "1.28.4"
############################
# role:cluster-addon
############################
# coredns 自动安装
#=================================这里更改为no=======================================#
dns_install: "no"
corednsVer: "1.9.3"
ENABLE_LOCAL_DNS_CACHE: false
dnsNodeCacheVer: "1.21.1"
# 设置 local dns cache 地址
LOCAL_DNS_CACHE: "169.254.20.10"
# metric server 自动安装
#=================================这里更改为no=======================================#
metricsserver_install: "no"
metricsVer: "v0.5.2"
# dashboard 自动安装
#=================================这里更改为no=======================================#
dashboard_install: "no"
dashboardVer: "v2.5.1"
dashboardMetricsScraperVer: "v1.0.8"
# prometheus 自动安装
#=================================这里更改为no=======================================#
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "35.5.1"
# nfs-provisioner 自动安装
#=================================这里更改为no=======================================#
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "v4.0.2"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"
# network-check 自动安装
network_check_enabled: false 
network_check_schedule: "*/5 * * * *"
############################
# role:harbor
############################
# harbor version，完整版本号
HARBOR_VER: "v2.1.3"
HARBOR_DOMAIN: "harbor.easzlab.io.local"
HARBOR_TLS_PORT: 8443
# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: true
# install extra component
HARBOR_WITH_NOTARY: false
HARBOR_WITH_TRIVY: false
HARBOR_WITH_CLAIR: false
HARBOR_WITH_CHARTMUSEUM: true
root@k8s-ha1:/etc/kubeasz/clusters/k8s-cluster1# 
5.5更改ansible 01.prepare.yml文件
root@deploy-harbor:/etc/kubeasz# vim playbooks/01.prepare.yml
#hosts 只留下这三个就可以
# [optional] to synchronize system time of nodes with 'chrony' 
- hosts:
  - kube_master
  - kube_node
  - etcd
  roles:
  - { role: os-harden, when: "OS_HARDEN|bool" }
  - { role: chrony, when: "groups['chrony']|length > 0" }
# to create CA, kubeconfig, kube-proxy.kubeconfig etc.
- hosts: localhost
  roles:
  - deploy
# prepare tasks for all nodes
- hosts:
  - kube_master
  - kube_node
  - etcd
  roles:
  - prepare
6. 部署K8S群
通过ansible脚本初始化环境及部署ks高可用集群
环境初始化
root@k8s-ha1:/etc/kubeasz# ./ezctl --help
Usage: ezctl COMMAND [args]
-------------------------------------------------------------------------------------
Cluster setups:
    list		             to list all of the managed clusters
    checkout    <cluster>            to switch default kubeconfig of the cluster
    new         <cluster>            to start a new k8s deploy with name 'cluster'
    setup       <cluster>  <step>    to setup a cluster, also supporting a step-by-step way
    start       <cluster>            to start all of the k8s services stopped by 'ezctl stop'
    stop        <cluster>            to stop all of the k8s services temporarily
    upgrade     <cluster>            to upgrade the k8s cluster
    destroy     <cluster>            to destroy the k8s cluster
    backup      <cluster>            to backup the cluster state (etcd snapshot)
    restore     <cluster>            to restore the cluster state from backups
    start-aio		             to quickly setup an all-in-one cluster with 'default' settings
Cluster ops:
    add-etcd    <cluster>  <ip>      to add a etcd-node to the etcd cluster
    add-master  <cluster>  <ip>      to add a master node to the k8s cluster
    add-node    <cluster>  <ip>      to add a work node to the k8s cluster
    del-etcd    <cluster>  <ip>      to delete a etcd-node from the etcd cluster
    del-master  <cluster>  <ip>      to delete a master node from the k8s cluster
    del-node    <cluster>  <ip>      to delete a work node from the k8s cluster
Extra operation:
    kcfg-adm    <cluster>  <args>    to manage client kubeconfig of the k8s cluster
Use "ezctl help <command>" for more information about a given command.
#查看脚本命令内容即可知道第几部都是在干什么
root@deploy-harbor:/etc/kubeasz# vim ezctl
 cat <<EOF
available steps:
    01  prepare            to prepare CA/certs & kubeconfig & other system settings 
    02  etcd               to setup the etcd cluster
    03  container-runtime  to setup the container runtime(docker or containerd)
    04  kube-master        to setup the master nodes
    05  kube-node          to setup the worker nodes
    06  network            to setup the network plugin
    07  cluster-addon      to setup other useful plugins
    90  all                to run 01~07 all at once
    10  ex-lb              to install external loadbalance for accessing k8s from outside
    11  harbor             to install a new harbor server or to integrate with an existed one
6.1 创建集群基础设置
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 01
遇到的问题
如果遇到卡导apt 进程了执行如下
sudo rm /var/lib/apt/lists/lock
sudo rm /var/cache/apt/archives/lock 
sudo rm /var/lib/dpkg/lock* 
sudo dpkg --configure -a sudo apt update
6.2 部署etcd集群
#所有需要拉取节点的需要配置这个  让containerd可以拉去harbor镜像
#这个是加到了ansible里面了  可以先加到剧本当中 在运行如果没有添加上 自己手动添加
root@k8s-ha1:/etc/kubeasz# vim roles/containerd/templates/config.toml.j2 
#这是在配置自己harbor仓库能让其它容器能拉取下来
#大概在147行
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.nbrhce.com"]
      endpoint = ["https://harbor.nbrhce.com"]
        [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.nbrhce.com".tls]
          insecure_skip_verify = true
 #这个用户是harbor内部用户
 [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.nbrhce.com".auth]
          username = "quyi"
          password = "Harbor12345"
#如果没有生成就自己手动去加
root@k8s-node1:~# vim /etc/containerd/config.toml
systemctl daemon-reload 
systemctl restart containerd
#测试是否能拉去镜像 这是之前harbor存放的镜像 如果你没有自己上传一个
ln -s /opt/kube/bin/crictl /usr/bin
crictl  pull harbor.nbrhce.com/demo/alpine:v1
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 02
#执行完测试
#etcd检查
export NODE_IPS="192.168.1.73 192.168.1.74 192.168.1.75"
for ip in ${NODE_IPS}; do ETCDCTL_API=3 /opt/kube/bin/etcdctl --endpoints=https://${ip}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem endpoint health;
#执行结果
https://192.168.1.76:2379 is healthy: successfully committed proposal: took = 8.57508ms
https://192.168.1.77:2379 is healthy: successfully committed proposal: took = 10.019689ms
https://192.168.1.78:2379 is healthy: successfully committed proposal: took = 8.723699ms
root@k8s-master1:~# crictl pull harbor.nbrhce.com/baseimages/pause:3.7 Image is up to date for sha256:221177c6082a88ea4f6240ab2450d540955ac6f4d5454f0e15751b653ebda165
6.3 部署运行时
#这个拉取不下来就初始化不会成功easzlab.io.local:5000/easzlab/pause:3.7
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 03
6.4 部署master集群
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 04
上面都部署成功了之后 在部署机器上查看以下节点
root@k8s-ha1:/etc/kubeasz# kubectl get nodes
NAME           STATUS                     ROLES    AGE    VERSION
192.168.1.70   Ready,SchedulingDisabled   master   102s   v1.24.2
192.168.1.71   Ready,SchedulingDisabled   master   102s   v1.24.2
6.5 部署node节点
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 05
root@deploy-harbor:/etc/kubeasz# kubectl get nodes
NAME           STATUS                     ROLES    AGE     VERSION
192.168.1.70   Ready,SchedulingDisabled   master   8m55s   v1.24.2
192.168.1.74   Ready                      node     50s     v1.24.2
6.6 部署calico
#需要更改calico镜像
#文件位置
root@k8s-ha1:/etc/kubeasz# vim roles/calico/templates/calico-v3.19.yaml.j2
docker tag easzlab.io.local:5000/calico/node:v3.19.4 harbor.nbrhce.com/baseimages/calico/node:v3.19.4
docker push harbor.nbrhce.com/baseimages/calico/node:v3.19.4
docker tag calico/pod2daemon-flexvol:v3.19.4 harbor.nbrhce.com/baseimages/calico-pod2daemon-flexvol:v3.19.4 
docker push harbor.nbrhce.com/baseimages/calico-pod2daemon-flexvol:v3.19.4
docker tag calico/cni:v3.19.4 harbor.nbrhce.com/baseimages/calico/cni:v3.19.4
docker push harbor.nbrhce.com/baseimages/calico/cni:v3.19.4
docker tag calico/kube-controllers:v3.19.4 harbor.nbrhce.com/baseimages/calico/kube-controllers:v3.19.4
docker push harbor.nbrhce.com/baseimages/calico/kube-controllers:v3.19.4
替换里面的image
#这个配置文件还可以更改地址段等等 以后用
root@k8s-ha1:/etc/kubeasz# vim roles/calico/templates/calico-v3.19.yaml.j2 
上面的镜像替换完毕之后在执行06
root@k8s-ha1:/etc/kubeasz# ./ezctl setup k8s-cluster1 06
root@deploy-harbor:/etc/kubeasz# kubectl get nodes
NAME           STATUS                     ROLES    AGE   VERSION
192.168.1.70   Ready,SchedulingDisabled   master   21h   v1.24.2
192.168.1.74   Ready                      node     21h   v1.24.2
root@deploy-harbor:/etc/kubeasz# kubectl get pods -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS        AGE
kube-system   calico-kube-controllers-68555f5f97-fks8p   1/1     Running   1 (99m ago)     21h
kube-system   calico-node-gdc8m                          1/1     Running   40 (9m8s ago)   21h
kube-system   calico-node-h5drr                          1/1     Running   36 (78s ago)    21h
7. 运行容器测试
root@deploy-harbor:/etc/kubeasz# kubectl run net-test1 --image=centos:7.9.2009 sleep 10000000 -n myserver
root@deploy-harbor:/etc/kubeasz# kubectl get pods -n myserver -o wide
NAME        READY   STATUS    RESTARTS   AGE   IP             NODE           NOMINATED NODE   READINESS GATES
net-test1   1/1     Running   0          66s   10.200.36.65   192.168.1.74   <none>           <none>
root@deploy-harbor:/etc/kubeasz# kubectl exec -it net-test1 bash -n myserver
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
#因为没有coredns 
[root@net-test1 /]# ping baidu.com
ping: baidu.com: Name or service not known
8. 添加master节点(扩容/缩容)
#先查看一下node节点所生成的配置文件 这个配置文件也是动态生成的
root@k8s-node1:~# vim /etc/kube-lb/conf/kube-lb.conf 
user root;
worker_processes 1;
error_log  /etc/kube-lb/logs/error.log warn;
events {
    worker_connections  3000;
stream {
    upstream backend {
#api-server地址
        server 192.168.1.70:6443    max_fails=2 fail_timeout=3s;
    server {
        listen 127.0.0.1:6443;
        proxy_connect_timeout 1s;
        proxy_pass backend;
#添加master节点
#在添加这个master节点的时候在模板文件中要配置harbor认证信息 否则会有报错拉取不下来镜像
root@deploy-harbor:/etc/kubeasz# ./ezctl add-master k8s-cluster1 192.168.1.77
#执行这个添加的操作他会把这个IP地址写在ansible的hosts文件里面
#删除就是del
root@deploy-harbor:/etc/kubeasz# ./ezctl del-master k8s-cluster1 192.168.1.77
9. 添加node节点(扩容/缩容)
root@deploy-harbor:/etc/kubeasz# ./ezctl add-node k8s-cluster1 192.168.1.77
#删除就是del
root@deploy-harbor:/etc/kubeasz# ./ezctl del-node k8s-cluster1 192.168.1.77
10. K8S升级（小版本）
10.1 官方位置
10.2 下载二进制安装包
#deploy服务器下载安装包  目前安装包没有找到
10.3 master升级
#在node节点上注释掉api-server地址 这样不会影响业务 如果有三个api-server注释掉其中一个就可以 每个节点都需要做
root@k8s-node1:~# vim /etc/kube-lb/conf/kube-lb.conf 
user root;
worker_processes 1;
error_log  /etc/kube-lb/logs/error.log warn;
events {
    worker_connections  3000;
stream {
    upstream backend {
#api-server地址
#        server 192.168.1.70:6443    max_fails=2 fail_timeout=3s;
    server {
        listen 127.0.0.1:6443;
        proxy_connect_timeout 1s;
        proxy_pass backend;
#然后需要停掉服务
systemctl stop kube-apiserver kube-sontroller-manager kube-scheduler kube-proxy kubelet
#拷贝二进制命令 拷贝到对应的节点上 master 复制到对应/usr/local/bin
kube-apiserver kube-sontroller-manager kube-scheduler kube-proxy kubelet
/usr/local/bin
#验证命令
/usr/local/bin/kube-apiserver --version
#查看是否修改相关的文件比如kube-proxy 
#还需要小心升级时候对应的api版本 如果版本不同的话 在重新搭建一套升级试试
vim /var/lib/kube-proxy/kube-proxy-config.yaml
api版本是指
apiVersion: kubeproxy.config.k8s.io/v1alpha1
#启动 master
systemctl start kube-apiserver kube-sontroller-manager kube-scheduler kube-proxy kubelet
kubectl get nodes 
#加载一下 node节点
systemctl reload kube-lb.service
#然后在把另外的两个注释掉 把新升级的开开
以上都是升级master
10.4 node升级
一定要驱除pod然后将服务停止然后替换二进制
#需要先把node上面的业务容器都要驱除走
kubectl drain 192.168.x.x --ignore-daemonsets --force
#检查是否驱除完毕
kubectl get pods -A -o wide
#停掉node服务
systemctl stop kubelet kube-proxy
#在部署节点上把kubelet kube-proxy 拷贝到node节点上
kubelet kube-proxy  /usr/local/bin
systemctl reload kube-lb.service
#在把api-server放开
vim /etc/kube-lb/conf/kube-lb.conf 
#再把所有二进制命令拷贝到 这样你部署的时候都是1.24.3的命令了
cp ./* /etc/kubeasz/bin/containerd-bin/
10.5 containerd升级runc升级客户端命令升级
#下载地址containerd
wget https://github.com/containerd/containerd/releases/download/v1.6.10/containerd-1.6.10-linux-amd64.tar.gz
#把containerd的新的命令拷贝过去
cp ./* /etc/kubeasz/bin/containerd-bin/
#下载地址 runc
wget https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
#下载客户端crictl ctr
#拷贝你所有准备的命令 拷贝的命令路径都是你containerd.service 里面写的路径
scp ./* 192.168.*:/usr/local/bin/
拷贝的时候会发现runc是拷贝不了的 因为node有占用
systemctl disable kubelet kube-proxy
systemctl disable containerd.setvice
reboot
然后在拷贝二进制命令
scp ./* 192.168.*:/usr/local/bin/
systemctl enable --now kubelet kube-proxy containerd.setvice
#重启一下node的kubelet
systemctl restart kubelet
#在把master节点上的containerd在升级一下
scp ./* 192.168.*:/usr/local/bin/
systemctl disable kubelet kube-proxy
systemctl disable containerd.setvice
reboot
systemctl enable --now kubelet kube-proxy containerd.setvice