Notice
Recent Posts
Recent Comments
Link
일 | 월 | 화 | 수 | 목 | 금 | 토 |
---|---|---|---|---|---|---|
1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 | 24 | 25 | 26 | 27 | 28 |
29 | 30 |
Tags
- Eclipse
- Android
- GIT
- 보조정렬
- Kotlin
- IntelliJ
- SQL
- Sqoop
- mapreduce
- vaadin
- xPlatform
- es6
- Spring
- window
- plugin
- R
- JavaScript
- Java
- react
- Python
- tomcat
- MSSQL
- hadoop
- SPC
- SSL
- mybatis
- NPM
- table
- 공정능력
- Express
Archives
- Today
- Total
DBILITY
apache sqoop2 설치 본문
반응형
설치 및 테스트는 oracle2hdfs를 테스트하기 위해서 2016년에 실시해 봤다.
머리속에 기록이 안되는 관계로 매뉴얼에 따라 정리한다.
아키텍쳐는 아래와 같다.
https://blogs.apache.org/sqoop/mediaresource/a04e8f6f-4d1e-4ee9-b726-af4e078599cf
[root@big-master ~]# adduser sqoop2 ; echo 'sqoop2' | passwd --stdin sqoop2 ; usermod -G datagroup sqoop2
Changing password for user sqoop2.
passwd: all authentication tokens updated successfully.
[root@big-master ~]# cd /usr/local/src
[root@big-master src]# wget http://apache.tt.co.kr/sqoop/1.99.7/sqoop-1.99.7-bin-hadoop200.tar.gz
[root@big-master src]# tar -zxpf sqoop-1.99.7-bin-hadoop200.tar.gz
[root@big-master src]# wget -P sqoop-1.99.7-bin-hadoop200/driver http://www.java2s.com/Code/JarDownload/ojdbc6/ojdbc6-11.2.0.2.0.jar.zip
[root@big-master src]# cd sqoop-1.99.7-bin-hadoop200/driver/
[root@big-master driver]# unzip ojdbc6-11.2.0.2.0.jar.zip
[root@big-master driver]# rm ojdbc6-11.2.0.2.0.jar.zip
[root@big-master driver]# cd /usr/local/src
[root@big-master src]# cp -rf sqoop-1.99.7-bin-hadoop200 /bigdata/
[root@big-master src]# chown -R sqoop2.sqoop2 /bigdata/sqoop-1.99.7-bin-hadoop200/
[root@big-master src]# ln -s /bigdata/sqoop-1.99.7-bin-hadoop200/ /sqoop
[root@big-master src]# vi /etc/profile
export SQOOP_HOME=/sqoop
export PATH=$PATH:$SQOOP_HOME/bin
:wq!
[root@big-master src]# source /etc/profile
[root@big-master src]# vi /hadoop/etc/hadoop/core-site.xml
<property>
<name>hadoop.proxyuser.sqoop2.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.sqoop2.groups</name>
<value>*</value>
</property>
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave1:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave2:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave3:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/core-site.xml big-slave4:/hadoop/etc/hadoop/core-site.xml
[root@big-master src]# vi /hadoop/etc/hadoop/mapred-site.xml
<property>
<name>mapreduce.child.java.opts</name>
<value>-Djava.security.egd=file:/dev/../dev/urandom</value>
</property>
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave1:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave2:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave3:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/mapred-site.xml big-slave4:/hadoop/etc/hadoop/mapred-site.xml
[root@big-master src]# vi /hadoop/etc/hadoop/container-executor.cfg
allowed.system.users=sqoop2##comma separated list of system users who CAN run applications
:wq!
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave1:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave2:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave3:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# rsync -zx /hadoop/etc/hadoop/container-executor.cfg big-slave4:/hadoop/etc/hadoop/container-executor.cfg
[root@big-master src]# vi /sqoop/conf/sqoop.properties
# Hadoop configuration directory
org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/hadoop/etc/hadoop/
:%s/@LOGDIR@/\/sqoop/g
:%s/@BASEDIR@/\/sqoop/g
:wq!
[root@big-master src]# vi /sqoop/bin/sqoop.sh
JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$HADOOP_HOME/lib/native -Djava.net.preferIPv4Stack=true"
SQOOP_SERVER_EXTRA_LIB=${BASEDIR}/driver
:wq!
[root@big-master src]# su - sqoop2
Last login: Fri Apr 20 22:56:24 KST 2018 on pts/1
[sqoop2@big-master ~]$ sqoop2-tool upgrade
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Sqoop tool executor:
Version: 1.99.7
Revision: 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
Compiled on Tue Jul 19 16:08:27 PDT 2016 by abefine
Running tool: class org.apache.sqoop.tools.tool.UpgradeTool
0 [main] INFO org.apache.sqoop.core.PropertiesConfigurationProvider - Starting config file poller thread
Tool class org.apache.sqoop.tools.tool.UpgradeTool has finished correctly.
[sqoop2@big-master ~]$ sqoop2-tool verify
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Sqoop tool executor:
Version: 1.99.7
Revision: 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
Compiled on Tue Jul 19 16:08:27 PDT 2016 by abefine
Running tool: class org.apache.sqoop.tools.tool.VerifyTool
0 [main] INFO org.apache.sqoop.core.SqoopServer - Initializing Sqoop server.
8 [main] INFO org.apache.sqoop.core.PropertiesConfigurationProvider - Starting config file poller thread
Verification was successful.
Tool class org.apache.sqoop.tools.tool.VerifyTool has finished correctly.
[sqoop2@big-master ~]$ sqoop.sh server start
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Starting the Sqoop2 server...
0 [main] INFO org.apache.sqoop.core.SqoopServer - Initializing Sqoop server.
8 [main] INFO org.apache.sqoop.core.PropertiesConfigurationProvider - Starting config file poller thread
Sqoop2 server started.
[sqoop2@big-master ~]$ sqoop2-shell
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Apr 20, 2018 11:05:55 PM java.util.prefs.FileSystemPreferences$1 run
INFO: Created user preferences directory.
Sqoop Shell: Type 'help' or '\h' for help.
sqoop:000> \h
For information about Sqoop, visit: http://sqoop.apache.org/
Available commands:
:exit (:x ) Exit the shell
:history (:H ) Display, manage and recall edit-line history
help (\h ) Display this help message
set (\st ) Configure various client options and settings
show (\sh ) Display various objects and configuration options
create (\cr ) Create new object in Sqoop repository
delete (\d ) Delete existing object in Sqoop repository
update (\up ) Update objects in Sqoop repository
clone (\cl ) Create new object based on existing one
start (\sta) Start job
stop (\stp) Stop job
status (\stu) Display status of a job
enable (\en ) Enable object in Sqoop repository
disable (\di ) Disable object in Sqoop repository
grant (\g ) Grant access to roles and assign privileges
revoke (\r ) Revoke access from roles and remove privileges
For help on a specific command type: help command
sqoop:000> set server --url http://big-master:12000/sqoop
Server is set successfully
sqoop:000> show version --all
client version:
Sqoop 1.99.7 source revision 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
server version:
Sqoop 1.99.7 source revision 435d5e61b922a32d7bce567fe5fb1a9c0d9b1bbb
Compiled by abefine on Tue Jul 19 16:08:27 PDT 2016
API versions:
[v1]
sqoop:000> show connector
+------------------------+---------+------------------------------------------------------------+----------------------+
| Name | Version | Class | Supported Directions |
+------------------------+---------+------------------------------------------------------------+----------------------+
| generic-jdbc-connector | 1.99.7 | org.apache.sqoop.connector.jdbc.GenericJdbcConnector | FROM/TO |
| kite-connector | 1.99.7 | org.apache.sqoop.connector.kite.KiteConnector | FROM/TO |
| oracle-jdbc-connector | 1.99.7 | org.apache.sqoop.connector.jdbc.oracle.OracleJdbcConnector | FROM/TO |
| ftp-connector | 1.99.7 | org.apache.sqoop.connector.ftp.FtpConnector | TO |
| hdfs-connector | 1.99.7 | org.apache.sqoop.connector.hdfs.HdfsConnector | FROM/TO |
| kafka-connector | 1.99.7 | org.apache.sqoop.connector.kafka.KafkaConnector | TO |
| sftp-connector | 1.99.7 | org.apache.sqoop.connector.sftp.SftpConnector | TO |
+------------------------+---------+------------------------------------------------------------+----------------------+
sqoop:000> show link
+------+----------------+---------+
| Name | Connector Name | Enabled |
+------+----------------+---------+
+------+----------------+---------+
sqoop:000> show job
+----+------+----------------+--------------+---------+
| Id | Name | From Connector | To Connector | Enabled |
+----+------+----------------+--------------+---------+
+----+------+----------------+--------------+---------+
sqoop:000>
sqoop:000> :x
[sqoop2@big-master ~]$ sqoop.sh server stop
Setting conf dir: /sqoop/bin/../conf
Sqoop home directory: /sqoop
Stopping the Sqoop2 server...
Sqoop2 server stopped.
[sqoop2@big-master ~]$ exit
logout
[root@big-master src]#
[root@big-master src]# vi /sqoop/bin/sqoop-service.sh
#!/bin/bash
start() {
source "/etc/profile"
sqoop2-server start
}
stop() {
source "/etc/profile"
sqoop2-server stop
}
case $1 in
start|stop) "$1" ;;
esac
exit 0
:wq!
[root@big-master src]# chown sqoop2.sqoop2 /sqoop/bin/sqoop-service.sh
[root@big-master src]# chmod 755 /sqoop/bin/sqoop-service.sh
[root@big-master src]# vi /etc/systemd/system/sqoop.service
[Unit]
Description = sqoop2 server ( ver. 1.99.7 )
After = network.target hadoop.service
Requires = network.target
[Service]
Type=oneshot
User=sqoop2
Group=sqoop2
ExecStart = /sqoop/bin/sqoop-service.sh start
ExecStop = /sqoop/bin/sqoop-service.sh stop
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
:wq!
[root@big-master src]# chmod 644 /etc/systemd/system/sqoop.service
[root@big-master src]# systemctl status sqoop.service
● sqoop.service - sqoop2 server ( ver. 1.99.7 )
Loaded: loaded (/etc/systemd/system/sqoop.service; disabled; vendor preset: disabled)
Active: active (exited) since Tue 2018-05-01 18:56:33 KST; 10s ago
Process: 2904 ExecStart=/sqoop/bin/sqoop-service.sh start (code=exited, status=0/SUCCESS)
Main PID: 2904 (code=exited, status=0/SUCCESS)
CGroup: /system.slice/sqoop.service
└─2944 /jdk/bin/java -Dsqoop.config.dir=/sqoop/bin/../conf -Djava.library.path=/hadoop/lib/native -classpath :/jdk/lib:/bigdata/lib:/sqoop/driver/ojdbc6-11.2.0.2.0.jar:/sqoop/server/lib/a...
May 01 18:56:27 big-master systemd[1]: Starting sqoop2 server ( ver. 1.99.7 )...
May 01 18:56:27 big-master sqoop-service.sh[2904]: stty: standard input: Inappropriate ioctl for device
May 01 18:56:27 big-master sqoop-service.sh[2904]: Setting conf dir: /sqoop/bin/../conf
May 01 18:56:27 big-master sqoop-service.sh[2904]: Sqoop home directory: /sqoop
May 01 18:56:28 big-master sqoop-service.sh[2904]: Starting the Sqoop2 server...
May 01 18:56:28 big-master sqoop-service.sh[2904]: 0 [main] INFO org.apache.sqoop.core.SqoopServer - Initializing Sqoop server.
May 01 18:56:28 big-master sqoop-service.sh[2904]: 8 [main] INFO org.apache.sqoop.core.PropertiesConfigurationProvider - Starting config file poller thread
May 01 18:56:33 big-master sqoop-service.sh[2904]: Sqoop2 server started.
May 01 18:56:33 big-master systemd[1]: Started sqoop2 server ( ver. 1.99.7 ).
[root@big-master src]# systemctl enable sqoop.service
Created symlink from /etc/systemd/system/multi-user.target.wants/sqoop.service to /etc/systemd/system/sqoop.service.
반응형
'bigdata > sqoop' 카테고리의 다른 글
sqoop2 etl hdfs to oracle (0) | 2018.05.01 |
---|---|
sqoop2 etl java API 테스트 (0) | 2018.04.30 |
sqoop2 etl oracle to hdfs (0) | 2018.04.22 |
Comments