Rancher K8s 구성 - 7(Private Registry 구성 - Harbor)

2024.02.15 - [Rancher k8s] - Rancher K8s 구성 - 1(DNS)

Rancher K8s 구성 - 1(DNS)

2024.02.15 - [Rancher k8s] - Rancher K8s 구성 - 2(HAproxy)

Rancher K8s 구성 - 2(HAproxy)

2024.02.20 - [Rancher k8s] - Rancher K8s 구성 - 3(Master, Worker)

Rancher K8s 구성 - 3(Master, Worker)

2024.02.21 - [Rancher k8s] - Rancher K8s 구성 - 4(MetalLB & Nginx Ingress 구성)

Rancher K8s 구성 - 4(MetalLB & Nginx Ingress 구성)

2024.02.21 - [Rancher k8s] - Rancher K8s 구성 - 5(Rancher UI 구성)

Rancher K8s 구성 - 5(Rancher UI 구성)

2024.02.21 - [Rancher k8s] - Rancher K8s 구성 - 6(Ceph Storage 구성)

Rancher K8s 구성 - 6(Ceph Storage 구성)

---

Harbor 설치

# namespace 생성
kubectl create namespace harbor
kubectl get ns | grep harbor
harbor                                   Active   9s

# harbor 구성
helm repo add harbor https://helm.goharbor.io
helm repo update
helm search repo harbor
NAME         	CHART VERSION	APP VERSION	DESCRIPTION
harbor/harbor	1.14.0       	2.10.0     	An open source trusted cloud native registry th...

# harbor 차트 다운로드
mkdir -p /home/ubuntu/k8s-opensource/harbor && cd /home/ubuntu/k8s-opensource/harbor
helm fetch harbor/harbor --untar
cd harbor

# sc 조회
kubectl get sc
NAME              PROVISIONER                  RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
rook-ceph-block   rook-ceph.rbd.csi.ceph.com   Delete          Immediate           true                   88m

# harbor 차트 수정
vi values.yaml
------------------------------------------------------------
ingress:
    hosts:
      core: harbor-ui.internal.xg.com  ##31번 라인

externalURL: https://harbor-ui.internal.xg.com  ##106번 라인

persistence:
  enabled: true
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      storageClass: "rook-ceph-block"
      accessMode: ReadWriteOnce
      size: 300Gi  ##196번 라인
    jobservice:
      jobLog:
        storageClass: "rook-ceph-block"
        accessMode: ReadWriteOnce
        size: 50Gi  ##204번 라인
    database:
      storageClass: "rook-ceph-block"
      accessMode: ReadWriteOnce
      size: 200Gi  ##213번 라인
    redis:
      storageClass: "rook-ceph-block"
      accessMode: ReadWriteOnce
      size: 10Gi  ##222번 라인
    trivy:
      storageClass: "rook-ceph-block"
      accessMode: ReadWriteOnce
      size: 50Gi  ##229번 라인

existingSecretAdminPasswordKey: HARBOR_ADMIN_PASSWORD
harborAdminPassword: "xgk8stest1234!@"  ##347번 라인
------------------------------------------------------------

# harbor 배포(5~6분 소요)
helm install harbor . -n harbor

# harbor Chart 배포 확인
helm ls -n harbor
NAME  	NAMESPACE	REVISION	UPDATED                                	STATUS  	CHART        	APP VERSION
harbor	harbor   	1       	2024-02-14 06:57:44.899913645 +0000 UTC	deployed	harbor-1.14.0	2.10.0

# harbor pod 확인
kubectl get pods -o wide -n harbor
NAME                                READY   STATUS    RESTARTS        AGE     IP           NODE             NOMINATED NODE   READINESS GATES
harbor-core-6589db5c85-gflpx        1/1     Running   3 (3m10s ago)   6m11s   10.42.3.28   r1-k8s-workre1   <none>           <none>
harbor-database-0                   1/1     Running   0               6m11s   10.42.4.27   r1-k8s-workre2   <none>           <none>
harbor-jobservice-7c89d5b56-tbj22   1/1     Running   5 (3m24s ago)   6m11s   10.42.5.27   r1-k8s-workre3   <none>           <none>
harbor-portal-5d4889f845-pkh28      1/1     Running   0               6m11s   10.42.5.26   r1-k8s-workre3   <none>           <none>
harbor-redis-0                      1/1     Running   0               6m11s   10.42.5.29   r1-k8s-workre3   <none>           <none>
harbor-registry-78f9dd9458-6sqtv    2/2     Running   0               6m11s   10.42.5.28   r1-k8s-workre3   <none>           <none>
harbor-trivy-0                      1/1     Running   0               6m11s   10.42.3.29   r1-k8s-workre1   <none>           <none>

# harbor ingress 확인
kubectl get ingress -n harbor
NAME             CLASS    HOSTS                       ADDRESS                                  PORTS     AGE
harbor-ingress   <none>   harbor-ui.internal.xg.com   10.10.X.204,10.10.X.205,10.10.X.206   80, 443   71s
---------------------------------------------------------------------------------------------------
# chart 수정 후 업그레이드 명령어 참고
helm upgrade -i harbor . -n harbor

# 삭제
helm uninstall harbor . -n harbor

# pvc 삭제
kubectl get pvc -n harbor
kubectl delete pvc -n harbor data-harbor-redis-0
kubectl delete pvc -n harbor data-harbor-trivy-0
kubectl delete pvc -n harbor database-data-harbor-database-0
kubectl delete pvc -n harbor harbor-jobservice
kubectl delete pvc -n harbor harbor-registry

# 도메인
    -도메인 : harbor-ui.internal.xg.com
    - 대상 IP : 10.10.X.1 (K8s Ingress Loadbalancer IP)
    
    
# Harbor UI 접속 확인 (https://harbor-ui.internal.xg.com/harbor)
    - ID/PW : **admin / xgk8stest1234!@

---

private Registry(Harbor)로 Image 업로드, 다운로드 테스트

# 도커 설치 및 경로 변경
curl -fsSL https://get.docker.com | sh

# 도커 정보 확인 : client - server, Docker Root Dir, Registry
docker info

# 도커 정보 확인 : Docker Engine - Community
docker version

# 도커 프로세스 내리기
sudo systemctl stop docker.service
sudo systemctl stop docker.socket
sudo systemctl status docker.service
sudo systemctl status docker.socket

# docker login 경우 기본 https로 요청이 가기 때문에 http로 가도록 허용
sudo su -
cat <<EOT>> /etc/docker/daemon.json
{
  "insecure-registries" : ["harbor-ui.internal.xg.com"]
}
EOT

# 재시작 후 확인
sudo systemctl start docker
sudo systemctl status docker.service
sudo systemctl status docker.socket

# docker login (hosts 등록 후 사용)
sudo docker login -u admin harbor-ui.internal.xg.com
Password: xgk8stest1234!@

Login Succeeded

# image 다운로드
sudo docker pull nginx

# image 확인
sudo docker images
REPOSITORY   TAG       IMAGE ID       CREATED        SIZE
nginx        latest    247f7abff9f7   3 months ago   187MB

# image tag 적용 (harbor에서 project 생성 후 진행)
# sudo docker tag [Image ID] [Harbor_URL]/[Project]/[Image Name]:[Tag]
sudo docker tag 247f7abff9f7 harbor-ui.internal.xg.com/test/test_nginx:1.0

# Harbor Registry 이미지 Push
sudo docker push harbor-ui.internal.xg.com/test/test_nginx:1.0
The push refers to repository [harbor-ui.internal.xg.com/test/test_nginx]
c4106aac4c13: Pushed
c57ad67cd261: Pushed
cbafde024849: Pushed
b88f5270d74a: Pushed
ff3d21334998: Pushed
4c6624954f4b: Pushed
ceb365432eec: Pushed
1.0: digest: sha256:3208d1a22a7baea147640ebbc351e35e8ca0d78e13a8df3c0a983127ffffe9bf size: 1778

# Harbor에서 이미지 확인

---

Rancher(k8s) Private Registry 설정 및 Pod 배포

# K8s Master에서 설정 진행 (Endpoint 및 인증 정보)
cd /etc/rancher/rke2/
sudo vi registries.yaml
mirrors:
  docker.io:
    endpoint:
      - "https://harbor-ui.internal.xg.com"
configs:
  "harbor-ui.internal.xg.com":
    auth:
      username: admin
      password: "xgk8stest1234!@"
    tls:
      #cert_file:
      #key_file:
      #ca_file:
      insecure_skip_verify: true

# RKE2 데몬 재실행
sudo systemctl restart rke2-server.service
systemctl status rke2-server.service

# Worker 노드에서 registries.yaml 파일 생성되었는지 확인 (생성되지 않았다면 동일하게 파일 생성 후 rke2 agent 데몬 재기동)
cd /etc/rancher/rke2/
sudo vi registries.yaml
mirrors:
  docker.io:
    endpoint:
      - "https://harbor-ui.internal.xg.com"
configs:
  "harbor-ui.internal.xg.com":
    auth:
      username: admin
      password: "xgk8stest1234!@"
    tls:
      #cert_file:
      #key_file:
      #ca_file:
      insecure_skip_verify: true

sudo systemctl restart rke2-agent.service
sudo systemctl status rke2-agent.service

# K8s에서 Private Registry의 이미지를 활용 배포 가능한지 확인(2~3분 소요)
kubectl create deployment ubuntu-test --image=harbor-ui.internal.xg.com/test/test_nginx:1.0 --replicas=1
kubectl get pods
NAME                           READY   STATUS    RESTARTS   AGE
ubuntu-test-66584c84c7-whb4p   1/1     Running   0          2m24s

# Pod 이미지 확인
kubectl describe pods ubuntu-test-66584c84c7-whb4p | grep Image
    Image:          harbor-ui.internal.xg.com/test/test_nginx:1.0
    Image ID:       harbor-ui.internal.xg.com/test/test_nginx@sha256:3208d1a22a7baea147640ebbc351e35e8ca0d78e13a8df3c0a983127ffffe9bf

# 확인 후 삭제
kubectl get deploy
kubectl delete deployment ubuntu-test

# pod 확인 후 삭제
kubectl get pods
kubectl describe pod ubuntu-test-6b75cc684c-qqjbr
kubectl delete pod ubuntu-test-6b75cc684c-qqjbr

# Harbor UI에서 로그 확인
    - DockerHub를 이용하지 않고 Private Registry를 통해 이미지를 저장 가능하다.