apache sqoop2 설치

설치 및 테스트는 oracle2hdfs를 테스트하기 위해서 2016년에 실시해 봤다.

머리속에 기록이 안되는 관계로 매뉴얼에 따라 정리한다.

아키텍쳐는 아래와 같다.

https://blogs.apache.org/sqoop/mediaresource/a04e8f6f-4d1e-4ee9-b726-af4e078599cf

[root@big-master ~]# adduser sqoop2 ; echo 'sqoop2' | passwd --stdin sqoop2 ; usermod -G datagroup sqoop2
Changing password for user sqoop2.
passwd: all authentication tokens updated successfully.

[root@big-master ~]# cd /usr/local/src
[root@big-master src]# wget http://apache.tt.co.kr/sqoop/1.99.7/sqoop-1.99.7-bin-hadoop200.tar.gz
[root@big-master src]# tar -zxpf sqoop-1.99.7-bin-hadoop200.tar.gz

[root@big-master src]# wget -P sqoop-1.99.7-bin-hadoop200/driver http://www.java2s.com/Code/JarDownload/ojdbc6/ojdbc6-11.2.0.2.0.jar.zip
[root@big-master src]# cd sqoop-1.99.7-bin-hadoop200/driver/
[root@big-master driver]# unzip ojdbc6-11.2.0.2.0.jar.zip
[root@big-master driver]# rm ojdbc6-11.2.0.2.0.jar.zip
[root@big-master driver]# cd /usr/local/src
[root@big-master src]# cp -rf sqoop-1.99.7-bin-hadoop200 /bigdata/
[root@big-master src]# chown -R sqoop2.sqoop2 /bigdata/sqoop-1.99.7-bin-hadoop200/
[root@big-master src]# ln -s /bigdata/sqoop-1.99.7-bin-hadoop200/ /sqoop
[root@big-master src]# vi /etc/profile
export SQOOP_HOME=/sqoop
export PATH=$PATH:$SQOOP_HOME/bin
:wq!
[root@big-master src]# source /etc/profile
[root@big-master src]# vi /hadoop/etc/hadoop/core-site.xml
<property>
  <name>hadoop.proxyuser.sqoop2.hosts</name>
  <value>*</value>
</property>
<property>
  <name>hadoop.proxyuser.sqoop2.groups</name>
  <value>*</value>
</property>
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave1:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave2:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave3:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave4:/hadoop/etc/hadoop/core-site.xml

[root@big-master src]# vi /hadoop/etc/hadoop/mapred-site.xml
<property>
  <name>mapreduce.child.java.opts</name>
  <value>-Djava.security.egd=file:/dev/../dev/urandom</value>
</property>
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave1:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave2:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave3:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave4:/hadoop/etc/hadoop/mapred-site.xml

[root@big-master src]# vi /hadoop/etc/hadoop/container-executor.cfg
allowed.system.users=sqoop2##comma separated list of system users who CAN run applications
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave1:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave2:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave3:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave4:/hadoop/etc/hadoop/container-executor.cfg

[root@big-master src]# vi /sqoop/conf/sqoop.properties
# Hadoop configuration directory
org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/hadoop/etc/hadoop/
:%s/@LOGDIR@/\/sqoop/g
:%s/@BASEDIR@/\/sqoop/g
:wq!

[root@big-master src]# vi /sqoop/bin/sqoop.sh
JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$HADOOP_HOME/lib/native -Djava.net.preferIPv4Stack=true"
SQOOP_SERVER_EXTRA_LIB=${BASEDIR}/driver
:wq!

[root@big-master src]# su - sqoop2
Last login: Fri Apr 20 22:56:24 KST 2018 on pts/1
[sqoop2@big-master ~]$ sqoop2-tool upgrade
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Sqoop tool executor:
        Version: 1.99.7
        Revision: 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
        Compiled on Tue Jul 19 16:08:27 PDT 2016 by abefine
Running tool: class org.apache.sqoop.tools.tool.UpgradeTool
0    [main] INFO  org.apache.sqoop.core.PropertiesConfigurationProvider  - Starting config file poller thread
Tool class org.apache.sqoop.tools.tool.UpgradeTool has finished correctly.
[sqoop2@big-master ~]$ sqoop2-tool verify
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Sqoop tool executor:
        Version: 1.99.7
        Revision: 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
        Compiled on Tue Jul 19 16:08:27 PDT 2016 by abefine
Running tool: class org.apache.sqoop.tools.tool.VerifyTool
0    [main] INFO  org.apache.sqoop.core.SqoopServer  - Initializing Sqoop server.
8    [main] INFO  org.apache.sqoop.core.PropertiesConfigurationProvider  - Starting config file poller thread
Verification was successful.
Tool class org.apache.sqoop.tools.tool.VerifyTool has finished correctly.

[sqoop2@big-master ~]$ sqoop.sh server start
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Starting the Sqoop2 server...
0    [main] INFO  org.apache.sqoop.core.SqoopServer  - Initializing Sqoop server.
8    [main] INFO  org.apache.sqoop.core.PropertiesConfigurationProvider  - Starting config file poller thread
Sqoop2 server started.
[sqoop2@big-master ~]$ sqoop2-shell
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Apr 20, 2018 11:05:55 PM java.util.prefs.FileSystemPreferences$1 run
INFO: Created user preferences directory.
Sqoop Shell: Type 'help' or '\h' for help.
sqoop:000> \h
For information about Sqoop, visit: http://sqoop.apache.org/

Available commands:
  :exit    (:x  ) Exit the shell
  :history (:H  ) Display, manage and recall edit-line history
  help     (\h  ) Display this help message
  set      (\st ) Configure various client options and settings
  show     (\sh ) Display various objects and configuration options
  create   (\cr ) Create new object in Sqoop repository
  delete   (\d  ) Delete existing object in Sqoop repository
  update   (\up ) Update objects in Sqoop repository
  clone    (\cl ) Create new object based on existing one
  start    (\sta) Start job
  stop     (\stp) Stop job
  status   (\stu) Display status of a job
  enable   (\en ) Enable object in Sqoop repository
  disable  (\di ) Disable object in Sqoop repository
  grant    (\g  ) Grant access to roles and assign privileges
  revoke   (\r  ) Revoke access from roles and remove privileges

For help on a specific command type: help command

sqoop:000> set server --url http://big-master:12000/sqoop
Server is set successfully
sqoop:000> show version --all
client version:
  Sqoop 1.99.7 source revision 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
server version:
  Sqoop 1.99.7 source revision 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
  Compiled by abefine on Tue Jul 19 16:08:27 PDT 2016
API versions:
  [v1]
sqoop:000> show connector
+------------------------+---------+------------------------------------------------------------+----------------------+
|          Name          | Version |                           Class                            | Supported Directions |
+------------------------+---------+------------------------------------------------------------+----------------------+
| generic-jdbc-connector | 1.99.7  | org.apache.sqoop.connector.jdbc.GenericJdbcConnector       | FROM/TO              |
| kite-connector         | 1.99.7  | org.apache.sqoop.connector.kite.KiteConnector              | FROM/TO              |
| oracle-jdbc-connector  | 1.99.7  | org.apache.sqoop.connector.jdbc.oracle.OracleJdbcConnector | FROM/TO              |
| ftp-connector          | 1.99.7  | org.apache.sqoop.connector.ftp.FtpConnector                | TO                   |
| hdfs-connector         | 1.99.7  | org.apache.sqoop.connector.hdfs.HdfsConnector              | FROM/TO              |
| kafka-connector        | 1.99.7  | org.apache.sqoop.connector.kafka.KafkaConnector            | TO                   |
| sftp-connector         | 1.99.7  | org.apache.sqoop.connector.sftp.SftpConnector              | TO                   |
+------------------------+---------+------------------------------------------------------------+----------------------+
sqoop:000> show link
+------+----------------+---------+
| Name | Connector Name | Enabled |
+------+----------------+---------+
+------+----------------+---------+
sqoop:000> show job
+----+------+----------------+--------------+---------+
| Id | Name | From Connector | To Connector | Enabled |
+----+------+----------------+--------------+---------+
+----+------+----------------+--------------+---------+
sqoop:000>
sqoop:000> :x
[sqoop2@big-master ~]$ sqoop.sh server stop
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Stopping the Sqoop2 server...
Sqoop2 server stopped.
[sqoop2@big-master ~]$ exit
logout
[root@big-master src]#
[root@big-master src]# vi /sqoop/bin/sqoop-service.sh
#!/bin/bash

start() {
        source "/etc/profile"
        sqoop2-server start
}

stop() {
        source "/etc/profile"
        sqoop2-server stop
}

case $1 in
        start|stop) "$1" ;;
esac

exit 0
:wq!
[root@big-master src]# chown sqoop2.sqoop2 /sqoop/bin/sqoop-service.sh
[root@big-master src]# chmod 755 /sqoop/bin/sqoop-service.sh
[root@big-master src]# vi /etc/systemd/system/sqoop.service
[Unit]
Description = sqoop2 server ( ver. 1.99.7 )
After = network.target hadoop.service
Requires = network.target

[Service]
Type=oneshot
User=sqoop2
Group=sqoop2
ExecStart = /sqoop/bin/sqoop-service.sh start
ExecStop = /sqoop/bin/sqoop-service.sh stop
RemainAfterExit=yes


[Install]
WantedBy=multi-user.target
:wq!
[root@big-master src]# chmod 644 /etc/systemd/system/sqoop.service
[root@big-master src]# systemctl status sqoop.service
● sqoop.service - sqoop2 server ( ver. 1.99.7 )
   Loaded: loaded (/etc/systemd/system/sqoop.service; disabled; vendor preset: disabled)
   Active: active (exited) since Tue 2018-05-01 18:56:33 KST; 10s ago
  Process: 2904 ExecStart=/sqoop/bin/sqoop-service.sh start (code=exited, status=0/SUCCESS)
 Main PID: 2904 (code=exited, status=0/SUCCESS)
   CGroup: /system.slice/sqoop.service
           └─2944 /jdk/bin/java -Dsqoop.config.dir=/sqoop/bin/../conf -Djava.library.path=/hadoop/lib/native -classpath :/jdk/lib:/bigdata/lib:/sqoop/driver/ojdbc6-11.2.0.2.0.jar:/sqoop/server/lib/a...

May 01 18:56:27 big-master systemd[1]: Starting sqoop2 server ( ver. 1.99.7 )...
May 01 18:56:27 big-master sqoop-service.sh[2904]: stty: standard input: Inappropriate ioctl for device
May 01 18:56:27 big-master sqoop-service.sh[2904]: Setting conf dir: /sqoop/bin/../conf
May 01 18:56:27 big-master sqoop-service.sh[2904]: Sqoop home directory: /sqoop
May 01 18:56:28 big-master sqoop-service.sh[2904]: Starting the Sqoop2 server...
May 01 18:56:28 big-master sqoop-service.sh[2904]: 0    [main] INFO  org.apache.sqoop.core.SqoopServer  - Initializing Sqoop server.
May 01 18:56:28 big-master sqoop-service.sh[2904]: 8    [main] INFO  org.apache.sqoop.core.PropertiesConfigurationProvider  - Starting config file poller thread
May 01 18:56:33 big-master sqoop-service.sh[2904]: Sqoop2 server started.
May 01 18:56:33 big-master systemd[1]: Started sqoop2 server ( ver. 1.99.7 ).
[root@big-master src]# systemctl enable sqoop.service
Created symlink from /etc/systemd/system/multi-user.target.wants/sqoop.service to /etc/systemd/system/sqoop.service.