1. 初始化自定义hosts

init-hosts.sh，记得修改自定义镜像仓库与当前环境的所有名称与IP

[root@hyemvuka01 k8s]# pwd
/root/package/k8s


[root@hyemvuka01 k8s]# cat init-hosts.sh 
#!/bin/bash
# init-hosts.sh
# 初始化 /etc/hosts 文件

set -e
set -o pipefail

echo ">>> 初始化 /etc/hosts 文件 ..."
cat > /etc/hosts <<'EOF'
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

10.0.42.19 harbor.bx.crpharm.com
10.133.179.20 vip.k8s.local
10.133.179.27     hyemvuka01     #master,etcd,worker
10.133.179.28     hyemvuka02     #master,etcd,worker
10.133.179.29     hyemvuka03     #master,etcd,worker
EOF

echo "✅ /etc/hosts 初始化完成"

2. 初始化系统参数

需要关闭iptables和firewalld防火墙之类的，还需要虚拟内存，同时配置为自启动

[root@hyemvuka01 k8s]# cat init-os.sh 
#!/bin/bash
# os-init.sh
# K8s 节点初始化脚本
# 功能：
# 1. 关闭 firewalld
# 2. 停用 iptables
# 3. 禁用 SELinux
# 4. 关闭 swap
# 5. 清空 /etc/sysctl.conf 并设置 swappiness=0
# 6. 配置 ipvs 和 br_netfilter 模块
# 7. 创建 systemd 服务自动加载模块和 sysctl 参数
# 8. 写入 Kubernetes 必要 sysctl 配置

set -e
set -o pipefail

echo ">>> 停止并禁用 firewalld ..."
if systemctl list-unit-files | grep -q firewalld.service; then
  systemctl stop firewalld || true
  systemctl disable firewalld || true
  systemctl status firewalld --no-pager
else
  echo "⚠️ 系统没有 firewalld 服务"
fi

echo ">>> 停止并禁用 iptables ..."
if systemctl list-unit-files | grep -q iptables.service; then
  systemctl stop iptables || true
  systemctl disable iptables || true
  systemctl status iptables --no-pager
else
  echo "⚠️ 系统没有 iptables 服务"
fi

echo ">>> 禁用 SELinux ..."
if command -v setenforce >/dev/null 2>&1; then
  setenforce 0 || true
fi
if [ -f /etc/selinux/config ]; then
  sed -ri 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
  sed -ri 's/^SELINUX=permissive/SELINUX=disabled/' /etc/selinux/config
fi
sestatus || true

echo ">>> 关闭 swap ..."
swapoff -a
if [ -f /etc/fstab ]; then
  sed -ri 's/.*swap.*/#&/' /etc/fstab
fi

echo ">>> 清空 /etc/sysctl.conf 并写入基础配置 ..."
> /etc/sysctl.conf
#echo "vm.swappiness=0" >> /etc/sysctl.conf
sysctl -p

echo ">>> 写入 Kubernetes 必要 sysctl 配置到 /etc/sysctl.d/k8s.conf ..."
cat > /etc/sysctl.d/k8s.conf <<'EOF'
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF

echo ">>> 配置 ipvs 内核模块脚本 ..."
yum -y install ipset ipvsadm
cat > /etc/sysconfig/modules/ipvs.modules <<'EOF'
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules

echo ">>> 配置 br_netfilter 模块脚本 ..."
cat > /etc/sysconfig/modules/br_netfilter.modules <<'EOF'
#!/bin/bash
modprobe br_netfilter
EOF
chmod 755 /etc/sysconfig/modules/br_netfilter.modules

echo ">>> 创建 k8s-modules.service ..."
cat > /etc/systemd/system/k8s-modules.service <<'EOF'
[Unit]
Description=Load Kubernetes required kernel modules
Before=network-pre.target
Wants=network-pre.target sysctl-k8s.service
DefaultDependencies=no

[Service]
Type=oneshot
ExecStart=/etc/sysconfig/modules/ipvs.modules
ExecStart=/etc/sysconfig/modules/br_netfilter.modules
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF

echo ">>> 创建 sysctl-k8s.service ..."
cat > /etc/systemd/system/sysctl-k8s.service <<'EOF'
[Unit]
Description=Apply Kubernetes required sysctl parameters
Before=network-pre.target
Wants=network-pre.target
After=k8s-modules.service
Requires=k8s-modules.service
DefaultDependencies=no

[Service]
Type=oneshot
ExecStart=/usr/sbin/sysctl -p /etc/sysctl.d/k8s.conf
RemainAfterExit=true

[Install]
WantedBy=multi-user.target
EOF

echo ">>> 重新加载 systemd 并启用服务 ..."
systemctl daemon-reload
systemctl enable --now k8s-modules.service
systemctl enable --now sysctl-k8s.service

echo "✅ 系统初始化完成"

3. 部署containerd容器运行时

因为他默认会将镜像给放在var目录，但是当前环境会严格控制每个大目录的空间，所以需要修改为/app目录，让它将镜像下载到这个目录

[root@hyemvuka01 k8s]# cat install-containerd.sh 
#!/bin/bash
# install-containerd.sh
# 安装 Containerd

set -e
set -o pipefail

mkdir -p /app/containerd

PKG_DIR="./containerd"
TAR_FILE="$PKG_DIR/cri-containerd-1.7.3-linux-arm64.tar.gz"
CONFIG_FILE="$PKG_DIR/config.toml"
RUNC_FILE="$PKG_DIR/runc"

# 1. 解压 containerd
if [[ -f "$TAR_FILE" ]]; then
  echo "解压 $TAR_FILE 到 / ..."
  tar -xf "$TAR_FILE" -C /
else
  echo "❌ 未找到 $TAR_FILE"
  exit 1
fi

# 2. 拷贝配置文件
if [[ -f "$CONFIG_FILE" ]]; then
  echo "复制 $CONFIG_FILE 到 /etc/containerd/"
  mkdir -p /etc/containerd
  cp -f "$CONFIG_FILE" /etc/containerd/config.toml
else
  echo "❌ 未找到 $CONFIG_FILE"
  exit 1
fi

# 3. 安装 runc
if [[ -f "$RUNC_FILE" ]]; then
  echo "复制 $RUNC_FILE 到 /usr/local/sbin/runc"
  install -m 755 "$RUNC_FILE" /usr/local/sbin/runc
else
  echo "❌ 未找到 $RUNC_FILE"
  exit 1
fi

# containerd 启动！
systemctl daemon-reload
systemctl enable --now containerd

echo "✅ Containerd 安装完成"

4. 安装k8s核心软件

当前安装为k8s 1.30.6版本，会自动从当前压缩包中一键安装

[root@hyemvuka01 k8s]# cat install-k8s.sh 
#!/bin/bash
# install.sh
# 安装当前目录下 k8s-install 里的所有 RPM 包

set -e
set -o pipefail

if ls ./k8s-install/*.rpm >/dev/null 2>&1; then
  echo "开始安装 ./k8s-install 下的所有 RPM 包..."
  yum install -y ./k8s-install/*.rpm
  echo "✅ 安装完成"
else
  echo "⚠️ 没有找到 ./k8s-install/*.rpm 包"
fi

5. 搭建高可用环境之一，keepalived

对外暴露的IP为虚拟IP，避免直接使用节点IP，因为节点可能会宕机，导致IP会失效，它能实现虚拟IP漂移到下一个节点，继续提供访问能力，避免宕机不可用，可以直接执行yum install -y keepalived命令直接安装，注意priority权重，以此递减10即可，还有virtual_router_id这个需要额外用一个新的，它为一个组，不能和现有其他环境的keepalived相同，当前配置的keepalived有脚本检查功能，如果此脚本检测失败不通过，keepalived将自动停止运行，此时虚拟IP将会漂移到下一个节点运行

yum install -y keepalived

5.1 master-conf

[root@hyemvuka01 k8s]# cat /etc/keepalived/keepalived.conf 
vrrp_script chk_apiserver {
   script "/etc/keepalived/check_apiserver.sh"
   interval 5
   weight -5
   fall 2
   rise 1
}
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 41
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass xxxxxxxxxxxxxxxxxxxxxxxxx
    }
    unicast_src_ip 10.133.179.27
    unicast_peer {
        10.133.179.28
        10.133.179.29
    }
    track_script {
        chk_apiserver
    }
    virtual_ipaddress {
        10.133.179.20
    }
}

5.2 back1-conf

vrrp_script chk_apiserver {
   script "/etc/keepalived/check_apiserver.sh"
   interval 5
   weight -5
   fall 2
   rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 41
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass xxxxxxxxxxxxxxxxxxxxxxxxx
    }
    unicast_src_ip 10.133.179.28
    unicast_peer {
        10.133.179.27
        10.133.179.29
    }
    track_script {
        chk_apiserver
    }
    virtual_ipaddress {
        10.133.179.20
    }
}

5.2 back2-conf

vrrp_script chk_apiserver {
   script "/etc/keepalived/check_apiserver.sh"
   interval 5
   weight -5
   fall 2
   rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 41
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 775BCEEF-3B2D-476D-9B54-AD0B15998705 
    }
    unicast_src_ip 10.133.179.29
    unicast_peer {
        10.133.179.27
        10.133.179.28
    }
    track_script {
        chk_apiserver
    }
    virtual_ipaddress {
        10.133.179.20
    }
}

6. 搭建高可用环境之二，haproxy

因为haproxy当前yum部署的版本实在过于老旧故使用源码安装，一般部署k8s高可用可以使用两种模式，一种是keepalived+nginx，一种是keepalived+haproxy，这个软件约等于是nginx，他的流程是访问虚拟IP:16443，这个16443是haproxy所占用，它会负载均衡到一个可用的k8s master节点的接口上

[root@hyemvuka03 haproxy]# pwd
/root/package/software/haproxy
[root@hyemvuka03 haproxy]# ll
total 5020
drwx------ 13 appuser appgrp    4096 Oct 27 10:48 haproxy-3.2.0
-rw-------  1 appuser appgrp 5125220 Oct 27 10:01 haproxy-3.2.0.tar.gz
-rw-------  1 appuser appgrp     959 Oct 27 10:01 haproxy.cfg
-rwx------  1 appuser appgrp    1141 Oct 27 10:45 install-haproxy.sh

[root@hyemvuka03 haproxy]# cat install-haproxy.sh 
#!/bin/bash
# install-haproxy.sh
# 编译安装 HAProxy 3.2.0

set -e
set -o pipefail

echo ">>> 安装编译依赖 ..."
yum install -y gcc make pcre-devel openssl-devel systemd-devel zlib-devel libtool lua-devel

# 进入 HAProxy 源码目录
cd ./haproxy-3.2.0 || { echo "❌ haproxy-3.2.0 目录不存在"; exit 1; }

echo ">>> 编译 HAProxy ..."
make TARGET=linux-glibc CPU=generic \
    USE_OPENSSL=1 \
    USE_PCRE=1 \
    USE_ZLIB=1 \
    USE_NS=1 \
    PREFIX=/usr

echo ">>> 安装 HAProxy ..."
make install PREFIX=/usr

# 上一个目录中有配置文件
mkdir -p /etc/haproxy/
cp ../haproxy.cfg /etc/haproxy/

echo "启动haproxy"
cat > /etc/systemd/system/haproxy.service <<'EOF'
[Unit]
Description=HAProxy Load Balancer
After=network.target

[Service]
Environment="CONFIG=/etc/haproxy/haproxy.cfg" "PIDFILE=/var/run/haproxy.pid"
ExecStart=/usr/sbin/haproxy -Ws -f $CONFIG -p $PIDFILE
ExecReload=/usr/sbin/haproxy -f $CONFIG -c -q
ExecReload=/bin/kill -USR2 $MAINPID
Restart=always
Type=notify

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload 
systemctl enable --now haproxy

echo "✅ HAProxy 安装完成"

[root@hyemvuka03 haproxy]#

7. 初始化k8s

来到master1节点执行命令，注意控制面板中的域名要使用hosts中的vip.k8s.local

[root@hyemvuka01 k8s]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

10.0.42.19 harbor.bx.crpharm.com
10.133.179.20 vip.k8s.local
10.133.179.27     hyemvuka01     #master,etcd,worker
10.133.179.28     hyemvuka02     #master,etcd,worker
10.133.179.29     hyemvuka03     #master,etcd,worker

[root@hyemvuka01 package]# cat k8s.info 
kubeadm init \
  --control-plane-endpoint "vip.k8s.local:16443" \
  --upload-certs \
  --apiserver-advertise-address=10.133.179.27 \
  --apiserver-bind-port=6443 \
  --pod-network-cidr=172.244.0.0/16 \
  --service-cidr=172.96.0.0/12 \
  --image-repository=harbor.bx.crpharm.com/k8s/arm64 \
  --kubernetes-version=1.30.6 \
  --node-name=hyemvuka01

x. 部署helm

到安装包中复制helm中将helm二进制命令复制到/usr/local/sbin中即可

x. 使用helm部署redis集群

x.x 部署redis-operator

将redis-operator部署到crpem的namespace中

helm install redis-operator ./redis-operator --create-namespace --namespace crpem

x.x 配置集群密码

它需要一个secret，来在部署的时候来指定当前集群为什么密码

cat > redis-secret.yaml<<'EOF'
kind: Secret
apiVersion: v1
metadata:
  name: redis-cluster-secret
  namespace: crpem
  annotations:
    kubesphere.io/creator: admin
data:
  password: ??????????????????????
type: Opaque
EOF

# 创建秘钥
kubectl create -f redis-secret.yaml

x.x 部署集群

部署六节点redis主从集群

apiVersion: redis.redis.opstreelabs.in/v1beta2
kind: RedisCluster
metadata:
  name: redis-cluster
  namespace: crpem
spec:
  clusterSize: 3
  clusterVersion: v6
  podSecurityContext:
    runAsUser: 1000
    fsGroup: 1000
  persistenceEnabled: true
  kubernetesConfig:
    image: harbor.bx.crpharm.com/k8s/arm64/opstree/redis:v6.2.19
    imagePullPolicy: IfNotPresent
    redisSecret:
      name: redis-cluster-secret
      key: password
  redisExporter:
    enabled: true
    image: harbor.bx.crpharm.com/k8s/arm64/opstree/redis-exporter:v1.45.0
  storage:
    volumeClaimTemplate:
      spec:
        # storageClassName: standard
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 2Gi
    nodeConfVolumeClaimTemplate:
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 2Gi

x.x 检查集群

先试用redis-cli -a xxxxxxxxxxxx登录集群，然后执行cluster info即可查看集群状态，可以看到六节点已经成功部署。

/data $ redis-cli -a xxxxxxxxxxxx
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> get a
(error) MOVED 15495 172.244.102.43:6379
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:3
cluster_my_epoch:1
cluster_stats_messages_ping_sent:155
cluster_stats_messages_pong_sent:166
cluster_stats_messages_sent:321
cluster_stats_messages_ping_received:163
cluster_stats_messages_pong_received:155
cluster_stats_messages_meet_received:3
cluster_stats_messages_received:321
127.0.0.1:6379>

x. 部署nfs存储类

为了给k8s提供一个存储位置，我们选择了nfs作为他的存储类，后续所有的数据都会放在此处

yum install -y nfs-utils rpcbind
systemctl enable --now rpcbind
systemctl enable --now nfs

mkdir /data/nfs
chmod 777 -R /data/nfs

# 配置路径，自行定义k8s集群ip增加安全性，防止恶意挂载
[root@hybxvdka01 nfs]# cat /etc/exports
/data/nfs 10.133.179.71(rw,sync,no_root_squash,no_subtree_check) \
          10.133.179.72(rw,sync,no_root_squash,no_subtree_check) \
          10.133.179.73(rw,sync,no_root_squash,no_subtree_check)

[root@hybxvdka01 nfs]# exportfs -r
[root@hybxvdka01 nfs]# showmount -e
Export list for hybxvdka01:
/data/nfs 10.133.179.71,10.133.179.72,10.133.179.73

# 到nfs-sc目录中，编辑nfs-subdir-external-provisioner.yaml配置文件，将其nfs的ip改为我们上述所指定的IP与存储目录即可

# 开始安装
kubectl create -f nfs-subdir-external-provisioner.yaml

x. 部署kubesphere控制面板

# 部署kubesphere安装器
kubectl create -f kubesphere-installer.yaml

# ks-install安装器完毕后，执行部署面板
kubectl create -f cluster-configuration.yaml

# 打印实时安装日志
kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f

# 安装完毕后任意节点的30880端口即可访问，默认账号：admin 密码：P@88w0rd

x. 安装velero还原器

通过velero还原器自动从信创DEV一键迁移到crpem服务器

# 将velero-v1.15.2-linux-arm64中的velero二进制复制到/usr/local/sbin目录即可
cp velero /usr/local/sbin

# 生成minio账号密码配置文件，以便从远程备份中还原内容
# minio 账号密码
cat >credentials-velero<<'EOF'
[default]
aws_access_key_id = 账号
aws_secret_access_key = 密码
EOF

# 开始安装，其中10.133.179.3:30051就是信创DEV的minio，此minio中包含了velero持续备份的数据在其中
velero install \
  --velero-pod-cpu-request 500m \
  --velero-pod-mem-request 0.5Gi \
  --velero-pod-cpu-limit 1000m \
  --velero-pod-mem-limit 1Gi \
  --node-agent-pod-cpu-request 500m \
  --node-agent-pod-mem-request 0.5Gi \
  --node-agent-pod-cpu-limit 1000m \
  --node-agent-pod-mem-limit 1Gi \
  --use-node-agent \
  --provider aws \
  --plugins harbor.bx.crpharm.com/k8s/arm64/velero/velero-plugin-for-aws:v1.11.0 \
  --bucket k8s-backup \
  --secret-file ./credentials-velero \
  --use-volume-snapshots=false \
  --backup-location-config region=cn-north-1,s3ForcePathStyle="true",s3Url=http://10.133.179.3:30051

# 获取备份
velero get backups

# 开始还原，将名称为crpem-back01的备份点，还原到当前纯新的环境中的crpem的namespace中
velero restore create --from-backup crpem-back01 --include-namespaces crpem

# 剩下的就是调整各种参数，例如将dev字样改成uat即可完成，它只能完成大约90%+的部署

如果觉得文章对你有用，请随意赞赏

先后顺序

https://tanqidi.com/archives/b1754656-a1e6-4ce5-9e64-13e3060728b8

作者

世界哪有真情在

发布于

2025-10-28

更新于