2024-06-01

Kubernetes 运维笔记

k8sops

一、使用kubeadm搭建 K8s 多节点集群

安装 containerd、kubeadm、kubelet、kubectl
设置内核模块
初始化 k8s

1.安装 Ubuntu

#!/bin/bash

iptables -A INPUT -p tcp -m multiport --dports 6443,2379,2380,10250 -j ACCEPT

# 关闭 swap 分区
swapoff -a		# 临时
sed -ri 's/.*swap.*/#&/' /etc/fstab  # 永久

# 设置 br_netfilter 模块
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter

EOF
# 加载模块
modprobe overlay		# 网络桥接模块
modprobe br_netfilter	# 联合文件系统模块

lsmod | grep -e br_netfilter -e overlay	# 检查

# 开启 bridge-nf-call-iptables ，设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
# 应用 sysctl 参数而不重新启动
sysctl --system

# 安装运行时 containerd
apt-get remove -y docker docker-engine docker.io containerd runc 
rm -rf /var/lib/docker
rm -rf /var/lib/containerd
apt-get install -y ca-certificates curl gnupg lsb-release
mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get install -y containerd.io
systemctl enable containerd
# 生成配置
containerd config default > /etc/containerd/config.toml
#把SystemdCgroup = false修改为：SystemdCgroup = true
sed -i 's/SystemdCgroup\ =\ false/SystemdCgroup\ =\ true/g' /etc/containerd/config.toml

#把sandbox_image = "k8s.gcr.io/"修改为：sandbox_image="registry.aliyuncs.com/google_containers/"
sed -i 's/sandbox_image\ =\ "k8s.gcr.io/sandbox_image\ =\ "registry.aliyuncs.com\/google_containers\/"/g' /etc/containerd/config.toml|grep sandbox_image

安装 containerd

# https://github.com/containerd/containerd/releases/latest	# 可以选择最新版
wget https://github.com/containerd/containerd/releases/download/v1.7.14/containerd-1.7.14-linux-amd64.tar.gz
sudo tar Cxzvf /usr/local containerd-1.6.2-linux-amd64.tar.gz

配置 systemd 管理

sudo mkdir /usr/local/lib/systemd/system/
sudo curl https://raw.githubusercontent.com/containerd/containerd/main/containerd.service >> /usr/local/lib/systemd/system/containerd.service

安装 runc

# https://github.com/opencontainers/runc/releases/latest	# 可以选择最新版
wget https://github.com/opencontainers/runc/releases/download/v1.1.12/runc.amd64
sudo install -m 755 runc.amd64 /usr/local/sbin/runc

安装 CNI 插件

# https://github.com/containernetworking/plugins/releases/latest	# 可以选择最新版
wget https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz
mkdir -p /opt/cni/bin
sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.4.1.tgz

安装 crictl

# https://github.com/kubernetes-sigs/cri-tools/releases/latest	# 可以选择最新版
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz
sudo tar Cxzvf /usr/local/bin/ crictl-v1.29.0-linux-amd64.tar.gz

cat >>  /etc/crictl.yaml << EOF
runtime-endpoint: unix:///var/run/containerd/containerd.sock
image-endpoint: unix:///var/run/containerd/containerd.sock
timeout: 10
debug: true
EOF

systemctl restart containerd

创建配置

sudo mkdir /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
sudo vim /etc/containerd/config.toml
---
# 设置 SystemdCgroup
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  ...
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
# 设置阿里镜像源
[plugins."io.containerd.grpc.v1.cri"]
	...
	sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.8"
---

sudo systemctl restart containerd

配置软件源，安装 kubelet、kubeadm、kubectl

sudo apt update && sudo apt upgrade -y
sudo apt-get install -y apt-transport-https ca-certificates curl
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt update && sudo apt install kubelet kubeadm kubectl -y
sudo apt install ntpdate proxychains4 -y	# ntpdate 同步时间，proxychains4 代理使用

安装 helm

curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
sudo apt-get install apt-transport-https --yes
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
sudo apt-get update
sudo apt-get install helm

配置时间服务器

sudo ntpdate -u pool.ntp.org

2.配置系统设置

基本设置

#!/bin/bash
# 关闭swap：
swapoff -a # 临时关闭
sed -ri 's/.*swap.*/#&/' /etc/fstab  #永久关闭

# 关闭selinux
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

# 关闭防火墙
iptables -F
iptables -X
systemctl stop firewalld.service
systemctl disable firewalld
cat > /etc/sysctl.conf << EOF
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
EOF
sysctl -p # 生效

cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

cat <<EOF | tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
EOF

sysctl --system # 生效

# 加载内核模块
modprobe br_netfilter  # 网络桥接模块
modprobe overlay # 联合文件系统模块
lsmod | grep -e br_netfilter -e overlay

配置 ipvs

sudo apt install -y ipvsadm ipset sysstat conntrack
# 加载模块
mkdir ~/k8s-init/
tee ~/k8s-init/ipvs.modules <<'EOF'
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_lc
modprobe -- ip_vs_lblc
modprobe -- ip_vs_lblcr
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- ip_vs_dh
modprobe -- ip_vs_fo
modprobe -- ip_vs_nq
modprobe -- ip_vs_sed
modprobe -- ip_vs_ftp
modprobe -- ip_tables
modprobe -- ip_set
modprobe -- ipt_set
modprobe -- ipt_rpfilter
modprobe -- ipt_REJECT
modprobe -- ipip
modprobe -- xt_set
modprobe -- br_netfilter
modprobe -- nf_conntrack
EOF

chmod 755 ~/k8s-init/ipvs.modules && sudo bash ~/k8s-init/ipvs.modules
sudo cp ~/k8s-init/ipvs.modules /etc/profile.d/ipvs.modules.sh
#查看对应的模块是否加载成功
lsmod | grep -e ip_vs -e nf_conntrack



#配置kubelet
cat >>  /etc/sysconfig/kubelet << EOF
# KUBELET_CGROUP_ARGS="--cgroup-driver=systemd"
KUBE_PROXY_MODE="ipvs"
EOF

修改 hosts 文件，并重启

10.10.0.2 master
10.10.0.3 node01
10.10.0.4 node02

如果是用虚拟机创建，此时可以复制三份

设置静态 ip

sudo vim /etc/netplan/00-installer-config.yaml
# This is the network config written by 'subiquity'
network:
  ethernets:
    ens33:
      dhcp4: false
      addresses: [10.10.0.2/24]	# 对应节点的 ip
      gateway4: 10.10.0.1
      nameservers:
        addresses: [10.10.0.1]

  version: 2

在三个节点分别执行

# Master 节点
sudo hostnamectl set-hostname master
# Node01 节点
sudo hostnamectl set-hostname node01
# Node02 节点
sudo hostnamectl set-hostname node02

3.初始化集群

如果下方初始化失败，可以重置

kubeadm reset
# 如果使用了 ipvs
ipvsadm --clear
# 如果创建了配置文件
rm $HOME/.kube/config

集群初始化

# Master 节点操作
sudo kubeadm version	# 查看版本
...GitVersion:"v1.29.3"...
# 如果想使用代理下载，需要设置，否则直接下载镜像
sudo systemctl set-environment HTTP_PROXY=127.0.0.1:1080
sudo systemctl set-environment HTTPS_PROXY=127.0.0.1:1080
# 或者
sudo systemctl set-environment HTTP_PROXY="socks5://127.0.0.1:1080"
sudo systemctl set-environment HTTPS_PROXY="socks5://127.0.0.1:1080"
# 然后重启
sudo systemctl restart containerd.service
# 下载镜像
sudo kubeadm config images pull --kubernetes-version=v1.29.3	# 下载镜像，注意版本与上方一致
# 初始化

sudo kubeadm init --apiserver-advertise-address=10.10.0.2 --kubernetes-version v1.29.3 --service-cidr=10.128.0.0/16 --pod-network-cidr=10.96.0.0/16
# 如果需要添加镜像站：--image-repository registry.aliyuncs.com/google_containers
# 如果出现 [kubelet-check] Initial timeout of 40s passed. 可能需要把代理关闭一下
...
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.10.0.2:6443 --token tr0a3f.0d8izupyvl10w9i7 \
        --discovery-token-ca-cert-hash sha256:85fa46e80ec7363213d397e381a862cfbb5c4f8dc8e3f10d9c4261eda0a0a2e2
...

# Master 节点执行

# 非 root 用户执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 如果使用 root 用户管理，需要：
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/profile
source /etc/profile


# Node 节点执行以加入集群
kubeadm join 10.10.0.2:6443 --token tr0a3f.0d8izupyvl10w9i7 \
        --discovery-token-ca-cert-hash sha256:85fa46e80ec7363213d397e381a862cfbb5c4f8dc8e3f10d9c4261eda0a0a2e2

设置网络

网络插件详情可看官方文档，此处选择 calico，此插件对 kubernetes 版本有要求，尽量在版本范围内。

mkdir -p k8s/calico && cd k8s/calico
wget https://github.com/projectcalico/calico/raw/master/manifests/calico.yaml
kubectl apply -f calico.yaml
kubectl get pods -n kube-system --watch	# 观察这几个 pod 是否启动，可能需要几分钟

Environment="HTTP_PROXY=http://127.0.0.1:1080"
Environment="HTTPS_PROXY=http://127.0.0.1:1080"