使用kube-key离线安装包部署K8S1.35+集群

1、离线包准备

1、在一个能上外网的机器上进行下载
export KKZONE=cn #初始化环境，适用于无法上外网的机器
curl -sfL https://get-kk.kubesphere.io | sh -   #下载kubekey
./package.sh config.yaml  #构建离线安装包
https://get-images.kubesphere.io/  #离线安装包的yaml配置从此页面生成
artifact.tgz  #此压缩包，即为离线安装包

2、开始执行安装

解压安装包
tar -xvf artifact.tgz && cd artifact
将kk命令解压到当前目录
tar -zxvf tools/amd64/kubekey-v4.0.8-linux-amd64.tar.gz -C  .
生成节点配置文件inventory.yaml并修改
./kk create inventory -o .
生成安装配置文件 config-v1.35.8.yaml并修改（注意K8S版本号与生成的离线配置文件里一致）
./kk create config --with-kubernetes v1.35.8 -o .
###安装前每个节点必须执行，不然会卡在apt那一步###
#原理：apt 装完包会触发 /etc/apt/apt.conf.d/99needrestart 的 Post-Invoke 钩子去调 needrestart，它默认弹交互菜单问"重启哪些服务
chmod -x /usr/lib/needrestart/apt-pinvoke 2>/dev/null || true
mkdir -p /etc/needrestart/conf.d
echo "\$nrconf{restart} = 'a'; \$nrconf{kernelhints} = 0;" > /etc/needrestart/conf.d/99-kubekey.conf
pkill -f 'sbin/needrestart' 2>/dev/null || true
开始安装
./kk create cluster -a kubekey-artifact.tgz -i inventory.yaml -c  config-v1.35.8.yaml

3、简化后的inventory.yaml文件参考

apiVersion: kubekey.kubesphere.io/v1
kind: Inventory
metadata:
  name: default
spec:
  hosts:
    zn-test01:
      connector:
        host: 192.168.0.150
      internal_ipv4: 192.168.0.150
    zn-test02:
      connector:
        host: 192.168.0.151
      internal_ipv4: 192.168.0.151
    zn-test03:
      connector:
        host: 192.168.0.152
      internal_ipv4: 192.168.0.152
    zn-test04:
      connector:
        host: 192.168.0.153
      internal_ipv4: 192.168.0.153
  groups:
    k8s_cluster:
      groups:
        - kube_control_plane
        - kube_worker
    kube_control_plane:
      hosts:
        - zn-test01
        - zn-test02
        - zn-test03
    kube_worker:
      hosts:
        - zn-test04
    etcd:
      hosts:
        - zn-test01
        - zn-test02
        - zn-test03
    image_registry:
      hosts:
        - zn-test01

4、简化后的config-v1.35.8.yaml文件参考

apiVersion: kubekey.kubesphere.io/v1
kind: Config
spec:
  # If set to "cn", online downloads will prioritize domestic sources when available.
  zone: "cn"
  kubernetes:
    # Specify the Kubernetes version to be installed.
    kube_version: v1.35.8
    # Specify the Helm version to be installed.
    helm_version: v3.18.5
    # Tag for the sandbox (pause) image used by pods.
    sandbox_image:
      tag: "3.10.1"
    control_plane_endpoint:
      # Supported HA types: local, kube-vip, haproxy.
      # If set to local, configure local hostname resolution as follows:
      #   - Control-plane nodes: 127.0.0.1  .kubernetes.control_plane_endpoint.host
      #   - Worker nodes:  .init_kubernetes_node  .kubernetes.control_plane_endpoint.host
      type: local
      # Kube-vip image tag used for static pod deployment (when type is kube-vip).
      kube_vip:
        image:
          tag: v0.7.2
      # HAProxy image tag used for static pod deployment (when type is haproxy).
      haproxy:
        image:
          tag: 2.9.6-alpine
  etcd:
    # Specify the etcd version to be installed.
    etcd_version: v3.6.6
  image_registry:
    # Image registry type to install. Supported: harbor, docker-registry.
    # Leave empty to skip installation (assuming an existing registry is available).
    type: "harbor"
    auth:
      registry: "dockerhub.kubekey.local"
      username: admin
      password: Harbor12345
      skip_tls_verify: false
      plain_http: false
    # Specify a VIP for image registry high availability. If set, enables HA.
    ha_vip: ""
    # ========== Image Registry - High Availability ==========
    # keepalived image tag for load balancing when multiple registry nodes exist.
    # keepalived_version: 2.3.4
    # ========== Image Registry - Harbor ==========
    # Image registry port settings. Usually you do not need to set them manually;
    # they are derived from image_registry.auth.registry and plain_http.
    # http_port: 80
    # https_port: 443
    # Harbor image tag (only valid if type is harbor).
    #harbor_version: v2.10.2
    # docker-compose binary
    #dockercompose_version: v2.20.3
    # ========== Image Registry - Docker Registry ==========
    # Docker Registry image tag (only valid if type is docker-registry).
    # docker_registry_version: 2.8.3
  cri:
    # Container runtime type. Supported: containerd, docker.
    container_manager: containerd
    # ========== CRI Tool ==========
    # crictl binary version.
    crictl_version: v1.35.0
    # ========== Docker Runtime ==========
    # Docker binary version.
    #docker_version: 29.6.2
    # cridockerd version (required for Kubernetes 1.24+).
    # cridockerd_version: v0.4.7
    # ========== Containerd Runtime ==========
    # containerd binary version (active only when container_manager is containerd).
    # Matches the containerd bundled in the docker static package above.
    containerd_version: v2.2.6
    # runc binary version (active only when container_manager is containerd).
    # runc_version: v1.3.6
    containerd:
      # 【关键】merge = 保留节点上已有的 /etc/containerd/config.toml，
      # 只把 kk 渲染的配置合并进去。
      # 原因：kk v4.0.8 的 config.toml 模板只输出 v1 字段 sandbox_image，
      # 而 containerd v2.2.6 只认 pinned_images.sandbox，
      # 不 merge 就会把节点上的 v2 版式覆盖回 v1，sandbox 镜像回落到 registry.k8s.io/pause。
      config_policy: merge
  cni:
    # CNI plugin type. Supported: calico, cilium, flannel, kubeovn, other
    type: calico
    # ========== Multi-CNI ==========
    # Multi-CNI type. Supported: multus, spiderpool, none.
    multi_cni: none
    # ========== Multi-CNI - Multus ==========
    # Image tag for Multus (configure if needed).
    # multus:
    #   image:
    #    tag: v4.3.0
    # ========== Multi-CNI - Spiderpool ==========
    # Spiderpool version (configure if needed).
    # spiderpool_version: v1.2.2
    # ========== Calico CNI ==========
    # Calico version (effective only when type is calico).
    calico_version: v3.32.2
    # Calico 地址池必须显式给出，kk 不会自动生成；cidr 要与默认 pod_cidr(10.233.64.0/18) 一致
    calico:
      values: |
        installation:
          calicoNetwork:
            ipPools:
              - cidr: 10.233.64.0/18
                encapsulation: VXLAN
                natOutgoing: Enabled
                nodeSelector: all()
    # ========== Cilium CNI ==========
    # Cilium version (effective only when type is cilium).
    #cilium_version: 1.20.1
    # ========== Flannel CNI ==========
    # Flannel version (effective only when type is flannel).
    #flannel_version: v0.28.9
    # ========== Kube-OVN CNI ==========
    # Kube-OVN version (effective only when type is kubeovn).
    #kubeovn_version: v1.16.2
  storage_class:
    # ========== Storage Class Configuration ==========
    # ========== Local/OpenEBS Storage Class ==========
    # Local storage class settings.
    local:
      enabled: true   # Enable the local storage class.
      default: true   # Set as the default storage class.
    # openebs/dynamic-localpv-provisioner helm chart version.
    localpv_provisioner_version: 4.4.0
    # ========== NFS Storage Class ==========
    nfs:
      enabled: false  # Enable the NFS storage class.
    # nfs-provisioner Helm chart version.
    #nfs_provisioner_version: 4.0.18
  dns:
    # CoreDNS settings.
    coredns:
      image:
        tag: v1.13.1
    # NodeLocalDNS settings.
    nodelocaldns:
      enabled: true
      image:
        tag: 1.26.4
  # the external images will to add in packages
  image_manifests: []

5、集群安装一半时报错，重新卸载K8S 再次执行上述安装

for ip in 192.168.0.150 192.168.0.151 192.168.0.152 192.168.0.153; do
  echo "== $ip =="
  ssh root@$ip 'kubeadm reset -f >/dev/null 2>&1
    rm -rf /etc/kubernetes/manifests/*.yaml /etc/kubernetes/admin.conf \
           /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf \
           /etc/kubernetes/scheduler.conf /etc/kubernetes/pki/etcd \
           /var/lib/etcd/* /var/lib/kubelet/* /etc/cni/net.d/* /var/lib/cni/* /root/.kube
    rm -f /etc/systemd/system/kubelet.service /lib/systemd/system/kubelet.service
    rm -rf /etc/systemd/system/kubelet.service.d
    systemctl daemon-reload; systemctl reset-failed
    echo "  LoadState=$(systemctl show kubelet -p LoadState --value)"'
done