sqoop,flume草稿

连xftp传sqoop压缩包到/opt/soft 目录下

cd opt/soft/
tar -zxvf sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz 
 mv sqoop-1.4.7.bin__hadoop-2.6.0 sqoop
cd sqoop/conf/
cp sqoop-env-template.sh sqoop-env.sh
vi sqoop-env-sh 
export HADOOP_COMMON_HOME=/opt/soft/hadoop
export HADOOP_MAPRED_HOME=/opt/soft/hadoop
export HBASE_HOME=/opt/soft/hbase
export HIVE_HOME=/opt/soft/hive

vi /etc/profile
export SQOOP_HOME=/opt/soft/sqoop
export PATH=$PATH:$SQOOP_HOME/bin
export HIVE_CONF_DIR=/opt/soft/hive/conf
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HIVE_HOME/lib/*
export CLASSPATH=.:$HIVE_HOME/lib

source /etc/profile
start-all.sh
zkServer.sh start
rm -rf /opt/soft/sqoop/lib/jackson-*
cp /opt/soft/hive/lib/mysql-connector-java-5.1.37.jar /opt/soft/sqoop/lib/
cp /opt/soft/hive/conf/hive-site.xml /opt/soft/sqoop/conf/
cp /opt/soft/hive/lib/hive-common-3.1.2.jar /opt/soft/sqoop/lib/
cp /opt/soft/hive/lib/hive-exec-3.1.2.jar /opt/soft/sqoop/lib/
cp /opt/soft/hive/lib/libthrift-0.9.3.jar /opt/soft/sqoop/lib/
cp /opt/soft/hive/lib/disruptor-3.3.6.jar /opt/soft/sqoop/lib/
cp /opt/soft/hive/lib/jackson-* /opt/soft/sqoop/lib/

useradd hadoop
su hadoop
测试sqoop是否能够正常连接MySQL数据库。
sqoop list-databases --connect jdbc:mysql://127.0.0.1:3306/ --username root -password 123456
创建MySQL数据库和数据表。 
mysql -uroot -p
create database sample;
use sample;
create table student (number char(9) primary key,name char(10));
insert into student values('01','zhangsan');
insert into student values('02','lisi');
insert into student values('03','wangwu');
select * from student;
exit
在hive中创建sample数据库和student数据表。
su root
cd /opt/soft/hive/conf
vi hive-site.xml  85改217
hive
create database sample;
show databases;
 use sample;
 create table student(number STRING,name STRING) row format delimited fields terminated by "|" stored as textfile;
exit;
列出MySQL数据库
sqoop list-databases -connect jdbc:mysql://localhost:3306/ -username root -password 123456
连接MySQL并列出sample库中的表
sqoop list-tables -connect jdbc:mysql://localhost:3306/sample -username root -password 123456
从MySQL中导出数据，并导入hive。
sqoop import --connect jdbc:mysql://192.168.217.80:3306/sample --username root --password 123456 --table student --fields-terminated-by '|' --delete-target-dir --num-mappers 1 --hive-import --hive-database sample --hive-table student
从hive中导出数据，并导入MySQL。
sqoop export --connect "jdbc:mysql://192.168.217.80:3306/sample?useUnicode=true&characterEncoding=utf-8" --username root --password 123456 --table student --input-fields-terminated-by '|' --export-dir /user/hive/warehouse/sample.db/student/*
查看数据是否导入浏览器 192.168.217.80:50070
Utilities  user  hive  warehouse  sample.db  student 
列出MySQL数据库中的所有数据库。
sqoop list-databases -connect jdbc:mysql://localhost:3306/ -username root -password 123456
连接MySQL并列出test数据库中的表。
sqoop list-tables -connect jdbc:mysql://localhost:3306/test -username root -password 123456
将关系型数据的表结构复制到hive中。
sqoop create-hive-table -connect jdbc:mysql://localhost:3306/test -table sqoop_test -username root -password Password 123456 -hive-table test
从关系数据库导入文件到hive中。
sqoop import -connect jdbc:mysql://localhost:3306/zxtest -username root -password 123456 -table sqoop_test -hive-import -hive-table s_test -m 1
从关系数据库导入MySQL中。
sqoop export -connect jdbc:mysql://localhost:3306/zxtest -username root -password root -table hive_test -export-dir /user/hive/warehouse/ new_test_partition
从数据库导出表的数据到HDFS文件中。
sqoop import -connect jdbc:mysql://localhost:3306/comperssion-username=hadoop-password=123456 -table HADOOP_USER_INFO -m 1 -target-dir /user/test
从数据库增量导入表数据到HDFS中
sqoop import -connect jdbc:mysql://localhost:3306/compression -username=hadoop -password=123456 -table HADOOP_USER_INFO -m 1 -target-dir /user/test -check-columnid -incremental append -last-value 3

解压、安装Flume
解压安装
解压、
cd /opt/soft
tar -zxvf apache-flume-1.6.0-bin.tar.gz 
安装Flume 配置Flume
mv apache-flume-1.6.0-bin flume
vim /etc/profile
内容:
export FLUME_HOME=/opt/soft/flume
export PATH=$PATH:$FLUME_HOME/bin

source /etc/profile

cd /opt/soft/flume/conf
ls一下 
cp flume-env.sh.template flume-env.sh
ls一下 看有没有生成flume-env.sh
vim flume-env.sh
内容:
export JAVA_HONE=/opt/soft/jdk

验证
flume-ng version

删除一些包
cd /opt/soft/hadoop/share/hadoop/common/lib
rm -rf guava-11.0.2.jar

在hdfs创建：hdfs dfs -mkdir -p /tmp/flume
然后给权限：hdfs dfs -chmod 777 /tmp/flume让它能够接收和发送数据

cd /opt/soft/flume
配置文件
vim simple-hdfs-flume.conf
内容为
a1.sources=r1
a1.sinks=k1
a1.channels=c1

a1.sources.r1.type=spooldir 
a1.sources.r1.spoolDir=/opt/soft/hadoop/logs/
a1.sources.r1.fileHeader=true 

a1.sinks.k1.type=hdfs 
a1.sinks.k1.hdfs.path=hdfs://master:9000/tmp/flume
a1.sinks.k1.hdfs.rollsize=1048760

a1.sinks.k1.hdfs.rollCount=0 
a1.sinks.k1.hdfs.rollInterval=900
a1.sinks.k1.hdfs.useLocalTimeStamp=true  

a1.channels.c1.type=file      
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.sources.r1.channels=c1 
a1.sinks.k1.channel=c1 

使用flume命令加载simple-hdfs-flume.conf配置信息，启动Flume传输数据
flume-ng agent --conf-file simple-hdfs-flume.conf --name a1

hdfs dfs -ls /tmp/flume看是否有传输文件