Hadoop 클러스터 - 하둡/스파크 명령어 테스트

하둡 명령어 테스트

# 하둡 확인
hdfs dfs -mkdir /test
hdfs dfs -ls /
Found 2 items
drwxr-xr-x   - ubuntu supergroup          0 2023-08-04 12:58 /test
drwxrwx---   - ubuntu supergroup          0 2023-08-04 12:55 /tmp

hdfs dfs -put /data/hadoop/LICENSE.txt /test/
hdfs dfs -ls /test
Found 1 items
-rw-r--r--   3 ubuntu supergroup      15217 2023-08-04 12:59 /test/LICENSE.txt

yarn jar /data/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.2.jar wordcount hdfs:///test/LICENSE.txt /test/output
hdfs dfs -text /test/output/*

## 삭제
hdfs dfs -rm /test/*
hdfs dfs -rm /test/output/*
hdfs dfs -rmdir /test/output
hdfs dfs -rmdir /test
hdfs dfs -ls /
Found 1 items
drwxrwx---   - ubuntu supergroup          0 2023-08-07 02:59 /tmp

# 로컬로 복사
hdfs dfs -get / /data/back/
백업

# Map Reduse 테스트
yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.2.jar pi 16 1000

# 속도테스트
yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.3.2-tests.jar TestDFSIO \
-write -nrFiles 50 -size 100MB

yarn jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.3.2-tests.jar TestDFSIO \
-read -nrFiles 50 -size 100MB


# API 디렉터리 조회
curl -s http://IP:50070/webhdfs/v1/?op=LISTSTATUS
{"FileStatuses":{"FileStatus":[
{"accessTime":0,"blockSize":0,"childrenNum":0,"fileId":16460,"group":"supergroup","length":0,"modificationTime":1691464321284,"owner":"ubuntu","pathSuffix":"system","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16386,"group":"supergroup","length":0,"modificationTime":1691462399661,"owner":"ubuntu","pathSuffix":"tmp","permission":"770","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16416,"group":"supergroup","length":0,"modificationTime":1691463268740,"owner":"ubuntu","pathSuffix":"user","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"}
]}}

# API 디렉터리 생성 후 조회
curl -i -X PUT "http://IP:50070/webhdfs/v1/test?op=MKDIRS&permission=755"
curl -s http://IP:50070/webhdfs/v1/?op=LISTSTATUS
{"FileStatuses":{"FileStatus":[
{"accessTime":0,"blockSize":0,"childrenNum":0,"fileId":16460,"group":"supergroup","length":0,"modificationTime":1691464321284,"owner":"ubuntu","pathSuffix":"system","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":0,"fileId":16505,"group":"supergroup","length":0,"modificationTime":1691474958068,"owner":"dr.who","pathSuffix":"test","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16386,"group":"supergroup","length":0,"modificationTime":1691462399661,"owner":"ubuntu","pathSuffix":"tmp","permission":"770","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16416,"group":"supergroup","length":0,"modificationTime":1691463268740,"owner":"ubuntu","pathSuffix":"user","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"}
]}}

# API 디렉터리 삭제 후 조회
curl -i -X DELETE "http://IP:50070/webhdfs/v1/test?op=DELETE"
curl -s http://IP:50070/webhdfs/v1/?op=LISTSTATUS
{"FileStatuses":{"FileStatus":[
{"accessTime":0,"blockSize":0,"childrenNum":0,"fileId":16460,"group":"supergroup","length":0,"modificationTime":1691464321284,"owner":"ubuntu","pathSuffix":"system","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16386,"group":"supergroup","length":0,"modificationTime":1691462399661,"owner":"ubuntu","pathSuffix":"tmp","permission":"770","replication":0,"storagePolicy":0,"type":"DIRECTORY"},
{"accessTime":0,"blockSize":0,"childrenNum":1,"fileId":16416,"group":"supergroup","length":0,"modificationTime":1691463268740,"owner":"ubuntu","pathSuffix":"user","permission":"755","replication":0,"storagePolicy":0,"type":"DIRECTORY"}
]}}

---

Spark 명령어 테스트

# Spark Word Count
spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster --driver-memory 512m --executor-memory 512m --executor-cores 1 $SPARK_HOME/examples/jars/spark-examples_2.12-3.3.2.jar 5
--------------------------------------------------
# pyspark 예제
cat <<EOT>> pyspark_example.py
from pyspark import SparkContext, SparkConf

conf = SparkConf()
conf.setMaster("yarn")
conf.setAppName("PJH Test")
sc = SparkContext(conf=conf)

print("="*100, "\n")
print(sc)
print("="*100, "\n")
EOT

# 실행
clear && spark-submit --master yarn --deploy-mode client pyspark_example.py