kubernetes二进制安装 -- 1.30.3

1、节点规划

10.202.99.34 master01
10.202.99.35 master02
10.202.99.36 master03
10.202.99.37 node01
10.202.99.100 vip

2、环境准备

2.1、关闭防火墙、selinux、swap和NetworkManager

# 关闭selinux
## 临时关闭
setenforce 0
## 永久关闭
sed -i 's/enforcing/disabled/' /etc/selinux/config

# 关闭防火墙
systemctl disable --now firewalld
systemctl stop firewalld
systemctl disable firewalld


# 关闭swap分区
sed -ri 's/.*swap.*/#&/' /etc/fstab 
swapoff -a 

# 关闭NetworkMannger
systemctl stop NetworkManager
systemctl disable NetworkManager

2.2、配置节点免密和时间同步

# 配置hosts主机名解析
# vim /etc/hosts
10.202.99.34 master01
10.202.99.35 master02
10.202.99.36 master03
10.202.99.37 node01

# 各节点之间免密通信。
# 生成密钥 (回车)
ssh-keygen -t rsa -q -N ''

# 分发公钥
for i in master01 master02 master03 node01;do ssh-copy-id -i ~/.ssh/id_rsa.pub $i;done
for i in master02 master03 node01;do scp -r /root/.ssh/  $i:/root/;done
# 分发hosts
for i in master02 master03 node01;do scp -r /etc/hosts  $i:/etc/;done

# 时间同步
yum install chrony -y

# vim /etc/chrony/chrony.conf
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp3.aliyun.com iburst

systemctl start chronyd
systemctl enable chronyd

# 配置centos7阿里源
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
# 配置epel阿里源
wget -O /etc/yum.repos.d/epel.repo https://mirrors.aliyun.com/repo/epel-7.repo
# yum makecahe

2.3、内核升级

# 下载内核安装包
# https://www.elrepo.org 目前不支持RHEL- 7，支持RHEL- 8和RHEL- 9
# 使用rpm包升级内核，rpm包下载地址
http://mirrors.coreix.net/elrepo-archive-archive/kernel/el7/x86_64/RPMS/
# centos内核升级需要下载三个rpm包（下载对应版本的包）
kernel
kernel-devel
kernel-headers

# 安装内核rpm包
# 安装内核包 (kernel)
rpm -ivh kernel-lt-5.4.278-1.el7.elrepo.x86_64.rpm
# 安装内核开发包 (kernel-devel)
rpm -ivh kernel-lt-devel-5.4.278-1.el7.elrepo.x86_64.rpm
# 安装内核头文件包 (kernel-headers)
rpm -ivh kernel-lt-headers-5.4.278-1.el7.elrepo.x86_64.rpm

# 验证内核版本
# 查看默认启动顺序
awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg

CentOS Linux (5.4.278-1.el7.elrepo.x86_64) 7 (Core)
CentOS Linux (3.10.0-1160.el7.x86_64) 7 (Core)
CentOS Linux (0-rescue-a66a00f1a66a00f1a66a00f1a66a00f1) 7 (Core)

# 默认启动的顺序是从0开始，新内核是从头插入（目前位置在0，而5.4.278的是在1），所以需要选择0
grub2-set-default 0  
# 重启系统
reboot
# 验证内核版本
uname -r
# 5.4.278-1.el7.elrepo.x86_64

2.4、安装IPVS

# 安装 IPVS
yum install -y conntrack-tools ipvsadm ipset conntrack libseccomp

# 加载 IPVS 模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr
ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in \${ipvs_modules}; do
/sbin/modinfo -F filename \${kernel_module} > /dev/null 2>&1
if [ $? -eq 0 ]; then
/sbin/modprobe \${kernel_module}
fi
done
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs

# 增加k8s转发配置并使其生效。（所有节点）
# /etc/sysctl.d/k8s.conf文件，添加如下内容：
# 内核参数

cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp.keepaliv.probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp.max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp.max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.top_timestamps = 0
net.core.somaxconn = 16384
EOF

# 立即生效
sysctl --system

2.5、docker安装、Containerd配置

# docker安装
# step 1: 安装必要的一些系统工具
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
# Step 2: 添加软件源信息
sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# Step 3
sudo sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# Step 4: 更新并安装Docker-CE
sudo yum makecache fast
sudo yum -y install docker-ce
# Step 5: 开启Docker服务和开机自启
systemctl start docker && systemctl enable docker

# Step 6: 配置阿里镜像加速（登录阿里云->容器镜像服务->镜像工具）
# 需要再添加 "exec-opts": ["native.cgroupdriver=systemd"]
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://d6mtathr.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
sudo systemctl daemon-reload && sudo systemctl restart docker && sudo systemctl status docker


#修改containerd配置
#本步骤node节点也要同样的修改
#备份源文件
cp /etc/containerd/config.toml /etc/containerd/config.toml.bak

####  Containerd 1.4 开始弃用runtime v1 了，考虑使用 runtime v2 需要添加一下配置

cat <<EOF > /etc/containerd/config.toml
version = 2
[plugins]
  [plugins."io.containerd.grpc.v1.cri"]
    sandbox_image = "registry.cn-guangzhou.aliyuncs.com/my_aliyund/pause:v3.9"
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
    [plugins."io.containerd.grpc.v1.cri".registry]
       config_path = "/etc/containerd/certs.d"
   [plugins."io.containerd.grpc.v1.cri".containerd]
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v2"
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
  [plugins."io.containerd.internal.v1.opt"]
    path = "/var/lib/containerd/opt"
EOF

sudo systemctl restart containerd && sudo systemctl status containerd && sudo systemctl enable containerd


###### 内核升级需要重启
#重启并检查
reboot  # 可安装IPVS和docker后再重启

3、集群证书

# 以下命令只需要在master01执行即可
# 安装证书生成工具
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64

# 设置执行权限
chmod a+x cfssljson_linux-amd64
chmod a+x cfssl_linux-amd64

# 移动到/usr/local/bin
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson

3.1、生成配置证书

mkdir -p /opt/cert/ca
cd /opt/cert/ca

cat > /opt/cert/ca/ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
           "expiry": "876000h"
      }
    }
  }
}
EOF

3.2、生成根证书请求文件

cat > /opt/cert/ca/ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names":[{
    "C": "CN",
    "ST": "GuangZhou",
    "L": "GuangZhou"
  }]
}
EOF

3.3、生成根证书

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

4、安装ETCD集群

4.1、节点规划

10.202.99.34 etcd01
10.202.99.35 etcd02
10.202.99.36 etcd03

4.2、创建ETCD集群证书

mkdir -p /opt/cert/etcd
cd /opt/cert/etcd

# hosts 为k8s节点和VIP的IP

cat > etcd-csr.json << EOF
{
    "CN": "etcd",
    "hosts": [
        "127.0.0.1",
        "10.202.99.34",
        "10.202.99.35",
        "10.202.99.36",
        "10.202.99.37",
        "10.202.99.100"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
          "C": "CN",
          "ST": "GuangZhou",
          "L": "GuangZhou"
        }
    ]
}
EOF

4.3、生成ETCD证书

cfssl gencert -ca=../ca/ca.pem -ca-key=../ca/ca-key.pem -config=