安装flume，简单使用学习记录

原创已于 2022-06-19 22:24:58 修改 · 951 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#flume #hadoop #big data

于 2021-12-23 23:41:57 首次发布

本文介绍如何使用Apache Flume进行日志数据的收集与传输。通过具体步骤展示Flume的安装配置过程，并实现从Netcat接收数据及从文件读取数据后发送至HDFS的全流程。该教程涵盖Flume基本组件的设置与应用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

[root@linux01 install]# tar -zxf flume-ng-1.6.0-cdh5.14.0.tar.gz -C ../soft

[root@linux01 soft]# mv apache-flume-1.6.0-cdh5.14.0-bin/ flume160

[root@linux01 soft]# cd flume160/

[root@linux01 flume160]# cd conf/

[root@linux01 conf]# cp flume-env.sh.template flume-env.sh

[root@linux01 conf]# vi ./flume-env.sh

22  export JAVA_HOME=/opt/soft/jdk180
25 # export JAVA_OPTS="-Xms3000m -Xmx3000m -Dcom.sun.management.jmxremote"

[root@linux01 conf]# yum install -y nc

[root@linux01 conf]# nc -lk 44444

[root@linux01 conf]# yum install telnet-server

[root@linux01 conf]# yum install telnet.*

[root@linux01 conf]# telnet localhost 44444

println("--------------------------------------------")

[root@linux01 kb15conf]# vi ./netcat-flume-logger.conf

a1.sources=r1
a1.channels=c1
a1.sinks=k1

a1.sources.r1.type=netcat
a1.sources.r1.bind=localhost
a1.sources.r1.port=44444

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.sinks.k1.type=logger

a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1                   

[root@linux01 flume160]# ./bin/flume-ng agent --name a1 --conf ./conf/ --conf-file ./conf/kb15conf/netcat-flume-logger.conf -Dflume.root.logger=INFO,console(先)

[root@linux01 conf]# telnet localhost 44444(后)

println("--------------------------------------------")

logger
[root@linux01 flume160]#./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-logger.conf -Dflume.root.logger=INFO,console(先)

[root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后)
[root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后)


[root@linux01 kb15conf]# vi ./file-flume-hdfs.conf 

a2.sources=r1
a2.channels=c1
a2.sinks=k1

a2.sources.r1.type=exec
a2.sources.r1.command=tail -f /opt/soft/flume160/conf/kb15conf/flumefile.log

a2.channels.c1.type=memory
a2.channels.c1.capacity=1000
a2.channels.c1.transactionCapacity=100

a2.sinks.k1.type=hdfs
a2.sinks.k1.hdfs.fileType=DataStream
a2.sinks.k1.hdfs.filePrefix=flumetohdfs
a2.sinks.k1.hdfs.fileSuffix=.txt
a2.sinks.k1.hdfs.path=hdfs://192.168.111.131:9000/kb15file

a2.sources.r1.channels=c1
a2.sinks.k1.channel=c1


上传到hdfs
[root@linux01 kb15conf]# hdfs dfsadmin -safemode leave

[root@linux01 flume160]# ./bin/flume-ng agent --name a2 --conf ./conf/ --conf-file ./conf/kb15conf/file-flume-hdfs.conf -Dflume.root.logger=INFO,console(先)

[root@linux01 kb15conf]# echo hello spark >> ./flumefile.log(后)
[root@linux01 kb15conf]# echo hello flume >> ./flumefile.log(后)