cloumon-elk 설치하기

참고하여 설치해보자. 시작하세요! 하둡프로그래밍의 저자가 근무하는 곳인가 보다.

책처럼 깔끔하다. 따라하면 되겠다.

 

http://gruter.github.io/cloumon-elk/

[root@big-master ~]# adduser gmonitor
[root@big-master ~]# passwd gmonitor
Changing password for user gmonitor.
New password:
BAD PASSWORD: The password contains the user name in some form
Retype new password:
passwd: all authentication tokens updated successfully.
[root@big-master ~]# usermod -G datagroup gmonitor
[root@big-master ~]# mkdir /bigdata/gmonitor
[root@big-master ~]# chown gmonitor.gmonitor /bigdata/gmonitor
[root@big-master ~]# su - gmonitor
Last login: Sun May  6 08:22:22 KST 2018 on pts/0
[gmonitor@big-master ~]$ ssh-keygen -t rsa -b 4096
Generating public/private rsa key pair.
Enter file in which to save the key (/home/gmonitor/.ssh/id_rsa):
Created directory '/home/gmonitor/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/gmonitor/.ssh/id_rsa.
Your public key has been saved in /home/gmonitor/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:foLNoLmh9IspKGi1EqIKdypmIem4xQUXyVKh3DxczRc gmonitor@big-master
The key's randomart image is:
+---[RSA 4096]----+
|   o+o.o  E.     |
| ..=oo  o .      |
|  +.*    .       |
|   o .           |
| .  . . S        |
|++ o o *         |
|X O * . = .      |
|XX O o   o       |
|O+* +.           |
+----[SHA256]-----+
[gmonitor@big-master ~]$ cd .ssh/
[gmonitor@big-master .ssh]$ ls
id_rsa  id_rsa.pub
[gmonitor@big-master .ssh]$ cp id_rsa.pub ./authorized_keys
[gmonitor@big-master .ssh]$ ssh-copy-id gmonitor@big-master
[gmonitor@big-master .ssh]$ ssh-copy-id gmonitor@big-slave1
[gmonitor@big-master .ssh]$ ssh-copy-id gmonitor@big-slave2
[gmonitor@big-master .ssh]$ ssh-copy-id gmonitor@big-slave3
[gmonitor@big-master .ssh]$ ssh-copy-id gmonitor@big-slave4

[root@big-master ~]# cd /usr/local/src/
[root@big-master src]# git clone https://github.com/gruter/cloumon-elk.git
Cloning into 'cloumon-elk'...
remote: Counting objects: 149, done.
remote: Total 149 (delta 0), reused 0 (delta 0), pack-reused 149
Receiving objects: 100% (149/149), 5.07 MiB | 2.27 MiB/s, done.
Resolving deltas: 100% (53/53), done.
[root@big-master src]# cd cloumon-elk/
[root@big-master cloumon-elk]# ls -al
total 6240
drwxr-xr-x.  8 root root    4096 May  6 07:23 .
drwxr-xr-x. 11 root root    4096 May  6 07:22 ..
drwxr-xr-x.  2 root root    4096 May  6 07:23 alert
-rwxr-xr-x.  1 root root    2111 May  6 07:23 download-modules
drwxr-xr-x.  4 root root      29 May  6 07:23 elasticsearch-template
-rw-r--r--.  1 root root    1580 May  6 07:23 env.sh
drwxr-xr-x.  8 root root    4096 May  6 07:23 .git
-rw-r--r--.  1 root root 6307840 May  6 07:23 gruter-system-metrics-agent-0.0.1.r6.tar
-rwxr-xr-x.  1 root root   15084 May  6 07:23 install
drwxr-xr-x.  2 root root    4096 May  6 07:23 kibana-data
drwxr-xr-x.  4 root root      29 May  6 07:23 kibana-template
-rw-r--r--.  1 root root     572 May  6 07:23 LICENSE.txt
drwxr-xr-x.  4 root root      29 May  6 07:23 logstash-template
-rwxr-xr-x.  1 root root    1236 May  6 07:23 make-tarball.sh
-rw-r--r--.  1 root root    5491 May  6 07:23 README.md
-rwxr-xr-x.  1 root root    2374 May  6 07:23 remote-es
-rwxr-xr-x.  1 root root    4127 May  6 07:23 sample-installer.sh
-rwxr-xr-x.  1 root root    6913 May  6 07:23 tools.sh
[root@big-master cloumon-elk]# git status
# On branch master
nothing to commit, working directory clean
[root@big-master cloumon-elk]# cd ..
[root@big-master src]# find ./ -name '.git'
./cloumon-elk/.git
[root@big-master src]# find . -name ".git" | xargs rm -rf
[root@big-master cloumon-elk]# ll
total 6228
drwxr-xr-x. 2 root root    4096 May  6 07:23 alert
-rwxr-xr-x. 1 root root    2111 May  6 07:23 download-modules
drwxr-xr-x. 4 root root      29 May  6 07:23 elasticsearch-template
-rw-r--r--. 1 root root    1580 May  6 07:23 env.sh
-rw-r--r--. 1 root root 6307840 May  6 07:23 gruter-system-metrics-agent-0.0.1.r6.tar
-rwxr-xr-x. 1 root root   15084 May  6 07:23 install
drwxr-xr-x. 2 root root    4096 May  6 07:23 kibana-data
drwxr-xr-x. 4 root root      29 May  6 07:23 kibana-template
-rw-r--r--. 1 root root     572 May  6 07:23 LICENSE.txt
drwxr-xr-x. 4 root root      29 May  6 07:23 logstash-template
-rwxr-xr-x. 1 root root    1236 May  6 07:23 make-tarball.sh
-rw-r--r--. 1 root root    5491 May  6 07:23 README.md
-rwxr-xr-x. 1 root root    2374 May  6 07:23 remote-es
-rwxr-xr-x. 1 root root    4127 May  6 07:23 sample-installer.sh
-rwxr-xr-x. 1 root root    6913 May  6 07:23 tools.sh
[root@big-master cloumon-elk]$ ./download-modules
Download the file from https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.1.1/elasticsearch-2.1.1.tar.gz
Untar the file
Rename elasticsearch-2.1.1 to elasticsearch
Download the file from https://download.elastic.co/kibana/kibana/kibana-4.3.1-linux-x64.tar.gz
Untar the file
Rename kibana-4.3.1-linux-x64 to kibana
Download the file from https://download.elastic.co/logstash/logstash/logstash-2.1.1.tar.gz
Untar the file
Rename logstash-2.1.1 to logstash
/usr/local/src/cloumon-elk /usr/local/src/cloumon-elk
Validating logstash-output-elasticsearch_java
Installing logstash-output-elasticsearch_java
Installation successful
Validating logstash-input-jmx
Installing logstash-input-jmx
Installation successful
Validating logstash-input-http_poller
Installing logstash-input-http_poller
Installation successful
Cloning into 'logstash-input-ganglia'...
remote: Counting objects: 259, done.
remote: Total 259 (delta 0), reused 0 (delta 0), pack-reused 259
Receiving objects: 100% (259/259), 40.94 KiB | 0 bytes/s, done.
Resolving deltas: 100% (102/102), done.
Branch huge-datagram set up to track remote branch huge-datagram from origin.
Switched to a new branch 'huge-datagram'
WARNING:  prerelease dependency on logstash-core (< 3.0.0, >= 2.0.0.beta2) is not recommended
WARNING:  open-ended dependency on logstash-codec-plain (>= 0) is not recommended
  if logstash-codec-plain is semantically versioned, use:
    add_runtime_dependency 'logstash-codec-plain', '~> 0'
WARNING:  open-ended dependency on logstash-devutils (>= 0, development) is not recommended
  if logstash-devutils is semantically versioned, use:
    add_development_dependency 'logstash-devutils', '~> 0'
WARNING:  open-ended dependency on gmetric (>= 0, development) is not recommended
  if gmetric is semantically versioned, use:
    add_development_dependency 'gmetric', '~> 0'
WARNING:  See http://guides.rubygems.org/specification-reference/ for help
  Successfully built RubyGem
  Name: logstash-input-ganglia
  Version: 2.0.4
  File: logstash-input-ganglia-2.0.4.gem
Validating ./logstash-input-ganglia/logstash-input-ganglia-2.0.4.gem
Installing logstash-input-ganglia
Installation successful
/usr/local/src/cloumon-elk
Deleting existing tarballs
Compressing Cloumon ELK
/usr/local/src/cloumon-elk /usr/local/src/cloumon-elk
Compressing hadoop templates
Compressing tajo templates
Compressing flume templates
Compressing hbase templates
Compressing system templates
Compressing zookeeper templates
Compressing elasticsearch templates
Success to compress templates
Success to moving home directory
/usr/local/src/cloumon-elk
[root@big-master cloumon-elk]# yum -y install expect
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
 * base: mirror.kakao.com
 * epel: ftp.jaist.ac.jp
 * extras: mirror.kakao.com
 * updates: mirror.kakao.com
Resolving Dependencies
--> Running transaction check
---> Package expect.x86_64 0:5.45-14.el7_1 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

================================================================================================================
 Package                  Arch                     Version                         Repository              Size
================================================================================================================
Installing:
 expect                   x86_64                   5.45-14.el7_1                   base                   262 k

Transaction Summary
================================================================================================================
Install  1 Package

Total download size: 262 k
Installed size: 566 k
Downloading packages:
expect-5.45-14.el7_1.x86_64.rpm                                                          | 262 kB  00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : expect-5.45-14.el7_1.x86_64                                                                  1/1
  Verifying  : expect-5.45-14.el7_1.x86_64                                                                  1/1

Installed:
  expect.x86_64 0:5.45-14.el7_1

Complete!
[root@big-master cloumon-elk]$ vi install
CLOUMON_HOME=/bigdata/$REMOTE_USER/cloumon
:wq!
[root@big-master cloumon-elk]$ ./install elasticsearch -s 192.168.100.180 -m 2g
Trying to create directory to deploy elasticsearch: /bigdata/gmonitor/cloumon/elk, /bigdata/gmonitor/cloumon/metrics/system
gmonitor@192.168.100.180's password:
Copying elasticsearch binary file to 192.168.100.180
gmonitor@192.168.100.180's password:
cloumon-elasticsearch.tar.gz                                                  100%   28MB  55.1MB/s   00:00
Extracting elasticsearch binary file at /bigdata/gmonitor/cloumon/elk
gmonitor@192.168.100.180's password:
Configuring elasticsearch options
gmonitor@192.168.100.180's password:
gmonitor@192.168.100.180's password:
Success to install elasticsearch.
Done
[root@big-master cloumon-elk]$ ./install kibana -s 192.168.100.180 -e "localhost"
Trying to create directory to deploy elasticsearch: /bigdata/gmonitor/cloumon/elk, /bigdata/gmonitor/cloumon/metrics/system
gmonitor@192.168.100.180's password:
Copying kibana binary file to 192.168.100.180
gmonitor@192.168.100.180's password:
cloumon-kibana.tar.gz                                                           100%   29MB  77.1MB/s   00:00
Extracting kibana binary file at /bigdata/gmonitor/cloumon/elk
gmonitor@192.168.100.180's password:
Configuring kibana options
gmonitor@192.168.100.180's password:
Success to install kibana
Done
[root@big-master cloumon-elk]$ vi hadoop.conf
namenode big-master
namenode big-slave1
journalnode big-master
journalnode big-slave1
journalnode big-slave2
datanode big-slave1
datanode big-slave2
datanode big-slave3
datanode big-slave4

[root@big-master cloumon-elk]$ ./install logstash -c hadoop -p jmx -s 192.168.100.180 -f ./hadoop.config -g development
Trying to create directory to deploy elasticsearch: /bigdata/gmonitor/cloumon/elk, /bigdata/gmonitor/cloumon/metrics/system
gmonitor@192.168.100.180's password:
Copying logstash binary file to 192.168.100.180
gmonitor@192.168.100.180's password:
cloumon-logstash.tar.gz                                                         100%   71MB  70.9MB/s   00:01
Extracting logstash binary file to 192.168.100.180
gmonitor@192.168.100.180's password:
Creating logstash configuration directory
gmonitor@192.168.100.180's password:
Creating temporary directory for logstash configuration
Preparing template files
/usr/local/src/cloumon-elk /usr/local/src/cloumon-elk
/usr/local/src/cloumon-elk
Setting logstash configurations
Creating directory for jmx configurations
gmonitor@192.168.100.180's password:
Setting logstash configurations for jmx
Copying logstash configuration for jmx
gmonitor@192.168.100.180's password:
namenode-big-master                                                             100%  259    58.2KB/s   00:00
namenode-big-slave1                                                             100%  259    36.2KB/s   00:00
journalnode-big-master                                                          100%  265    72.8KB/s   00:00
journalnode-big-slave1                                                          100%  265    46.6KB/s   00:00
journalnode-big-slave2                                                          100%  265    73.2KB/s   00:00
datanode-big-slave1                                                             100%  259    45.5KB/s   00:00
datanode-big-slave2                                                             100%  259    71.1KB/s   00:00
datanode-big-slave3                                                             100%  259    45.4KB/s   00:00
datanode-big-slave4                                                             100%  259    71.1KB/s   00:00
Copying logstash configuration files
gmonitor@192.168.100.180's password:
development-hadoop-jmx-to-es.config                                             100% 1194     1.6MB/s   00:00
Deleting temporary files
Success to install logstash
Done
[root@big-master cloumon-elk]$ vi namenodes.config
big-master
big-slave1
[root@big-master cloumon-elk]$ ./install logstash -s 192.168.100.180 -c hadoop-poller -p ganglia -f ./namenodes.config -k -g development
Trying to create directory to deploy elasticsearch: /bigdata/gmonitor/cloumon/elk, /bigdata/gmonitor/cloumon/metrics/system
gmonitor@192.168.100.180's password:
Creating logstash configuration directory
gmonitor@192.168.100.180's password:
Creating temporary directory for logstash configuration
Preparing template files
/usr/local/src/cloumon-elk /usr/local/src/cloumon-elk
/usr/local/src/cloumon-elk
Setting logstash configurations
Copying logstash configuration files
gmonitor@192.168.100.180's password:
development-hadoop-poller-ganglia-to-es.config                                  100% 3438     1.2MB/s   00:00
Deleting temporary files
Success to install logstash
Done

[gmonitor@big-master cloumon-elk]$ /bigdata/gmonitor/cloumon/elk/elasticsearch/bin/start-es.sh
elasticsearch is running now.
[gmonitor@big-master cloumon-elk]$ jps
30993 Jps
30965 Elasticsearch
[gmonitor@big-master cloumon-elk]$ ./kibana-data/restore.sh 192.168.100.180:5911 ./kibana-data/hadoop-ganglia.json development
restore response: 200 OK

[gmonitor@big-master cloumon-elk]$ /bigdata/gmonitor/cloumon/elk/logstash/bin/start-logstash.sh -g development -c hadoop -p jmx
development-hadoop-jmx-logstash is started

[gmonitor@big-master cloumon-elk]$ /bigdata/gmonitor/cloumon/elk/kibana/bin/start-kibana.sh
cat: /bigdata/gmonitor/cloumon/elk/kibana/kibana.pid: No such file or directory
kibana is started
[gmonitor@big-master cloumon-elk]$   log   [10:01:45.536] [info][listening] Server running at http://0.0.0.0:5931