Rancher K8s 구성 - 3(Master, Worker)

2024.02.15 - [Rancher k8s] - Rancher K8s 구성 - 1(DNS)

Rancher K8s 구성 - 1(DNS)

2024.02.15 - [Rancher k8s] - Rancher K8s 구성 - 2(HAproxy)

Rancher K8s 구성 - 2(HAproxy)

2024.02.21 - [Rancher k8s] - Rancher K8s 구성 - 4(MetalLB & Nginx Ingress 구성)

---

Rancher k8s 설치 공통

# 호스트네임 설정
sudo hostnamectl set-hostname 호스트네임

# 스왑 제거 및 주석 처리
sudo swapoff -a
sudo vi /etc/fstab
#/swap.img      none    swap    sw      0       0

# 커널 설정
sudo su -
cat <<EOT>> /etc/sysctl.conf
net.core.netdev_max_backlog=250000
net.core.somaxconn=65535
net.ipv4.tcp_max_syn_backlog=16384
net.ipv4.tcp_fin_timeout=12
net.ipv4.tcp_tw_reuse=1

net.core.rmem_max = 268435456
net.core.wmem_max = 268435456
net.core.rmem_default=10485760
net.core.wmem_default=10485760
net.ipv4.tcp_rmem=4096 87380 134217728
net.ipv4.tcp_wmem=4096 87380 134217728

vm.max_map_count=262144
net.ipv4.ip_local_port_range=1024 65535
net.ipv4.tcp_max_tw_buckets=500000

vm.overcommit_memory=2
vm.overcommit_ratio=90

fs.inotify.max_user_watches=2099999999           # pod 로그 확인 시 파일 제한 해제
fs.inotify.max_user_instances=2099999999
fs.inotify.max_queued_events=2099999999
EOT

# openfile 설정
cat <<EOT>> /etc/security/limits.conf
*           soft     nofile          1000000
*           hard     nofile          1000000
EOT

# apt
sudo apt update && sudo apt upgrade -y

# K8s Master 노드 추가 구성 (ubuntu 계정) - v1.27.8+rke2r1
curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_CHANNEL=v1.27.8+rke2r1 sh -
[INFO]  finding release for channel v1.27.8+rke2r1
[INFO]  using v1.27.8+rke2r1 as release
[INFO]  downloading checksums at https://github.com/rancher/rke2/releases/download/v1.27.8+rke2r1/sha256sum-amd64.txt
[INFO]  downloading tarball at https://github.com/rancher/rke2/releases/download/v1.27.8+rke2r1/rke2.linux-amd64.tar.gz
[INFO]  verifying tarball
[INFO]  unpacking tarball file to /usr/local

# rke2 config 디렉토리 생성
sudo mkdir -p /etc/rancher/rke2

---

Rancher K8s Master 구성

# rke2 config 파일 설정
sudo vi /etc/rancher/rke2/config.yaml
write-kubeconfig-mode: "0600"
tls-san:
  - r1.k8s.control-plain.internal.xg.com
  - 10.10.X.200

cni:
  - cilium

disable:
  - rke2-canal
  - rke2-kube-proxy

# rke2-server 실행 및 상태 확인
sudo systemctl enable rke2-server.service
sudo systemctl start rke2-server.service
sudo systemctl status rke2-server

# kubectl 파일 복사
sudo cp /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/

# kubeconfig 환경 변수 설정
mkdir -p $HOME/.kube
export VIP=r1.k8s.control-plain.internal.xg.com
sudo cat /etc/rancher/rke2/rke2.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

cat <<EOT>> ~/.bashrc
export KUBECONFIG=$HOME/.kube/config
EOT

source ~/.bashrc

# K8s Cluster 조회
kubectl cluster-info
Kubernetes control plane is running at https://r1.k8s.control-plain.internal.xg.com:6443
CoreDNS is running at https://r1.k8s.control-plain.internal.xg.com:6443/api/v1/namespaces/kube-system/services/rke2-coredns-rke2-coredns:udp-53/proxy

# 버전 확인
kubectl version
Client Version: version.Info{Major:"1", Minor:"27", GitVersion:"v1.27.8+rke2r1", GitCommit:"66fee42707cd7f5a89f1987f7cb81b02dd19161c", GitTreeState:"clean", BuildDate:"2023-11-15T21:08:03Z", GoVersion:"go1.20.11 X:boringcrypto", Compiler:"gc", Platform:"linux/amd64"}
Kustomize Version: v5.0.1
Server Version: version.Info{Major:"1", Minor:"27", GitVersion:"v1.27.8+rke2r1", GitCommit:"66fee42707cd7f5a89f1987f7cb81b02dd19161c", GitTreeState:"clean", BuildDate:"2023-11-15T21:08:03Z", GoVersion:"go1.20.11 X:boringcrypto", Compiler:"gc", Platform:"linux/amd64"}

# Master 토큰 조회
sudo cat /var/lib/rancher/rke2/server/node-token
K109476a614ac381c44cf6ad3793dce9e2d6094eca77b9484ebd662c34dde72bc52::server:eaa5f7c63ce1b4a60bfe6b643f1286a6

---

Rancher K8s Master 추가 가입

# rke2 config 파일 설정
sudo vi /etc/rancher/rke2/config.yaml
server: https://r1.k8s.control-plain.internal.xg.com:9345
token: K109476a614ac381c44cf6ad3793dce9e2d6094eca77b9484ebd662c34dde72bc52::server:eaa5f7c63ce1b4a60bfe6b643f1286a6
write-kubeconfig-mode: "0600"
tls-san:
  - r1.k8s.control-plain.internal.xg.com
  - 10.10.X.200

# rke2-server 실행
sudo systemctl enable rke2-server.service
sudo systemctl start rke2-server.service
sudo systemctl status rke2-server

# kubectl 파일 복사
sudo cp /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/

# kubeconfig 환경 변수 설정
mkdir -p $HOME/.kube
export VIP=r1.k8s.control-plain.internal.xg.com
sudo cat /etc/rancher/rke2/rke2.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

cat <<EOT>> ~/.bashrc
export KUBECONFIG=$HOME/.kube/config
EOT

source ~/.bashrc

# cluster info 정상 조회 확인
kubectl cluster-info
Kubernetes control plane is running at https://r1.k8s.control-plain.internal.xg.com:6443
CoreDNS is running at https://r1.k8s.control-plain.internal.xg.com:6443/api/v1/namespaces/kube-system/services/rke2-coredns-rke2-coredns:udp-53/proxy

# 버전 확인
kubectl version
Client Version: version.Info{Major:"1", Minor:"27", GitVersion:"v1.27.8+rke2r1", GitCommit:"66fee42707cd7f5a89f1987f7cb81b02dd19161c", GitTreeState:"clean", BuildDate:"2023-11-15T21:08:03Z", GoVersion:"go1.20.11 X:boringcrypto", Compiler:"gc", Platform:"linux/amd64"}
Kustomize Version: v5.0.1
Server Version: version.Info{Major:"1", Minor:"27", GitVersion:"v1.27.8+rke2r1", GitCommit:"66fee42707cd7f5a89f1987f7cb81b02dd19161c", GitTreeState:"clean", BuildDate:"2023-11-15T21:08:03Z", GoVersion:"go1.20.11 X:boringcrypto", Compiler:"gc", Platform:"linux/amd64"}

# master 노드 조회
kubectl get nodes -o wide
NAME             STATUS   ROLES                       AGE     VERSION          INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
r1-k8s-master1   Ready    control-plane,etcd,master   13m     v1.27.8+rke2r1   10.10.X.201   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-master2   Ready    control-plane,etcd,master   29s     v1.27.8+rke2r1   10.10.X.202   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-master3   Ready    control-plane,etcd,master   9m30s   v1.27.8+rke2r1   10.10.X.203   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1

---

Rancher K8s Worker 구성

# rke2 config 파일 설정
sudo vi /etc/rancher/rke2/config.yaml
server: https://r1.k8s.control-plain.internal.xg.com:9345
token: K109476a614ac381c44cf6ad3793dce9e2d6094eca77b9484ebd662c34dde72bc52::server:eaa5f7c63ce1b4a60bfe6b643f1286a6
write-kubeconfig-mode: "0600"
tls-san:
  - r1.k8s.control-plain.internal.xg.com
  - 10.10.X.200

# rke2-server 실행
sudo systemctl enable rke2-agent.service
sudo systemctl start rke2-agent.service
systemctl status rke2-agent

# master에서 worker 노드 추가 확인
kubectl get nodes -o wide
NAME             STATUS   ROLES                       AGE     VERSION          INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
r1-k8s-master1   Ready    control-plane,etcd,master   18m     v1.27.8+rke2r1   10.10.X.201   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-master2   Ready    control-plane,etcd,master   5m40s   v1.27.8+rke2r1   10.10.X.202   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-master3   Ready    control-plane,etcd,master   14m     v1.27.8+rke2r1   10.10.X.203   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-workre1   Ready    <none>                      46s     v1.27.8+rke2r1   10.10.X.204   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-workre2   Ready    <none>                      36s     v1.27.8+rke2r1   10.10.X.205   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1
r1-k8s-workre3   Ready    <none>                      24s     v1.27.8+rke2r1   10.10.X.206   <none>        Ubuntu 22.04.3 LTS   5.15.0-92-generic   containerd://1.7.7-k3s1