Airflow local 설치

설치환경 - ubuntu 20.04

# 시스템 업그레이드
sudo apt update && sudo apt upgrade -y

# mysql 설치
sudo apt install -y mysql-server
sudo systemctl start mysql
sudo systemctl enable mysql
sudo systemctl status mysql
● mysql.service - MySQL Community Server
     Loaded: loaded (/lib/systemd/system/mysql.service; enabled; vendor preset: enabled)
     Active: active (running) since Fri 2023-02-10 05:30:27 UTC; 38s ago
   Main PID: 162789 (mysqld)
     Status: "Server is operational"
      Tasks: 39 (limit: 154187)
     Memory: 366.5M
     CGroup: /system.slice/mysql.service
             └─162789 /usr/sbin/mysqld

Feb 10 05:30:26 IDC-Rack systemd[1]: Starting MySQL Community Server...
Feb 10 05:30:27 IDC-Rack systemd[1]: Started MySQL Community Server.

# mysql 접속
sudo mysql -u root -p
Enter password: 
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 8
Server version: 8.0.32-0ubuntu0.20.04.2 (Ubuntu)

Copyright (c) 2000, 2023, Oracle and/or its affiliates.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

# Airflow용 계정 생성
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '패스워드';
mysql> create user 'airflow'@'localhost' identified by 'airflow!';
mysql> create user 'airflow'@'%' identified by 'airflow!';
mysql> CREATE DATABASE airflow;
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| airflow            |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
5 rows in set (0.01 sec)
mysql> exit

# 추가 패키지 설치
sudo apt install -y vim ssh openssh-server software-properties-common apt-transport-https wget
sudo apt install -y python3 freetds-bin krb5-user ldap-utils libsasl2-2 libsasl2-modules libssl-dev locales lsb-release sqlite3 sasl2-bin unixodbc  python3-pip python3-testresources mysql-client libmysqlclient-dev python3-mysqldb

# 디렉터리 생성
mkdir -p /data/airflow && cd /data/airflow

# Airflow 환경 설정 적용
# 환경설정
export AIRFLOW_HOME=/data/airflow
export AIRFLOW_VERSION=2.3.4
export PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
export CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"

export AIRFLOW_HOME=/data/airflow
export AIRFLOW_VERSION=2.5.1
export PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
export CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-no-providers-${PYTHON_VERSION}.txt"

# 설정 확인 
echo $AIRFLOW_HOME
/data/airflow

# Airflow 설치
pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"

# mysql 모듈 설치
pip install apache-airflow[mysql]

# 초기 폴더와 파일 만들기
airflow db init
ll
total 452
drwxr-xr-x 3 root root     82 Feb 10 06:03 ./
drwxr-xr-x 3 root root     21 Feb 10 05:55 ../
-rw-r--r-- 1 root root  49491 Feb 10 06:02 airflow.cfg
-rw-r--r-- 1 root root 401408 Feb 10 06:03 airflow.db
drwxr-xr-x 3 root root     23 Feb 10 06:02 logs/
-rw-r--r-- 1 root root   4707 Feb 10 06:02 webserver_config.py

# DB mysql로 변경 
sudo vi airflow.cfg

# 기존설정 주석 처리 후 변경
[CORE]
...
#executor = SequentialExecutor    #24번줄
executor = LocalExecutor
...

[database]     
...
#sql_alchemy_conn = sqlite:////data/airflow/airflow.db    #195번줄
sql_alchemy_conn = mysql://airflow:airflow!@localhost:3306/airflow
...

# mysql airflow DB 만들어진 테이블 접근 권한 부여
sudo mysql -u root -p'패스워드'
mysql> grant all privileges on airflow.* to airflow@localhost;
mysql> grant all privileges on airflow.* to airflow@'%';
mysql> flush privileges;

# db 초기화
airflow db init

---

Airflow 시작하기

# 초기 folder(dags, plugins)만들기 
cd $AIRFLOW_HOME
mkdir dags plugins

# user create password는 나중에 쳐야함
airflow users create \
    --username airflow \
    --firstname airflow \
    --lastname airflow \
    --role Admin \
    --email 이메일

패스워드:airflow!

# webserver 실행
airflow webserver --port 8080 -D

# scheduler 실행
airflow scheduler -D

접속정보 : http://서버IP:8080

airflow / airflow!

 

---

Airflow 설정값 변경

설정변경(sudo vi airflow.cfg) - airflow.db 삭제금지!!

# 설정변경
sudo vi airflow.cfg

# db 초기화
airflow db init

# 웹, 스케쥴러 삭제
pgrep scheduler | xargs kill -9
pgrep gunicorn | xargs kill -9

# user create password는 나중에 쳐야함
airflow users create \
    --username airflow \
    --firstname airflow \
    --lastname airflow \
    --role Admin \
    --email 이메일

패스워드:airflow!

# webserver 실행
airflow webserver --port 8080 -D

# scheduler 실행
airflow scheduler -D