The recent project team has the need to tap the stream log to collect, learn a bit flume and install successfully. The relevant information to record a bit.
1) Download flume1.5 version
wget http://www.apache.org/dyn/closer.cgi/flume/1.5.0.1/apache-flume-1.5.0.1-bin.tar.gz
2) Unzip the flume1.5
TAR-ZXVF apache-flume-1.5.0.1-bin.tar.gz
3) Configure Environment variables
JDK is installed
Export Flume_home=/xxx/xx/apache-flume-1.5.0.1-bin
Export path= $FLUME _home/bin: $PATH </p><p align= "left" >
4) Configure Conf related files </p><p align= "left" >
4.1) configuration flume-env.sh mainly set up a bit java_home
4.2) Configuration Log4j.properties
If the test environment is commented out Flume.root.logger=info,logfile Select Flume.root.logger=debug,console Print the log to the console
4.3) Configure flume-conf.properties This file name can be arbitrarily changed when you run the command, specify the properties file you created yourself
#set Agent name for A1 sources name for R1 sinks name for K1 channels name for C1
A1.sources = R1
A1.sinks = K1
A1.channels = C1
Sources component type executes Linux command for exec
A1.sources.r1.type = Exec
A1.sources.r1.command = Tail-f/home/hadoop/flume/flume/conf/source.txt (big difference in size tail-f solves a big problem for us)
Sinks Component Type is logger
A1.sinks.k1.type = Logger
Channels component type is memory
A1.channels.c1.type = Memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
Connect the sources, the sinks and the pipes.
A1.sources.r1.channels = C1
A1.sinks.k1.channel = C1
5) Run the command in the Flume directory
Bin/flume-ng agent-n a1-f test/source-tail-sink-logger.properties--conf conf
A preliminary example is completed. At present, our production environment is two nodes to the Metaq inside the production of data. Metaq Customizing a sink (custom sink see following code)
Remember to put the Metaq related jar under Flume/lib Gecko-1.1.4.jar Metamorphosis-client-1.4.6.2.jar Metamorphosis-commons-1.4.6.2.jar Zkclient-0.3.jar Zookeeper-3.4.3.jar
A1.sources = R1
A1.sinks = K1
A1.channels = C1
A1.sources.r1.type = Exec
A1.sources.r1.command = Tail- F/home/hadoop/flume/flume/conf/source.txt
A1.sinks.k1.type = com. XX.flume.sink.MetaQSink
A1.sinks.k1.sink.zkConnect = 0.0.0.0:2181,0.0.0.0:2181,0.0.0.0:2181
A1.sinks.k1.sink.zkRoot =/meta (this directory must be written dead)
A1.sinks.k1.sink.topic = XXXX
A1.sinks.k1.sink.batchSize = 20000
#a1. Channels.c1.type = Memory
#a1. channels.c1.capacity = 1000000
#a1. channels.c1.transactionCapacity = 100000
A1.channels.c1.type = File
A1.channels.c1.checkpointDir =/home/hadoop/flume/flume/channel/checkpoint
A1.channels.c1.dataDirs =/home/hadoop/flume/flume/channel/data
A1.sources.r1.channels = C1
A1.sinks.k1.channel = C1
Custom Sink Code
</pre><pre name= "code" class= "java" >package com.jd.flume.sink;import com.taobao.metamorphosis.Message; Import Com.taobao.metamorphosis.client.messagesessionfactory;import Com.taobao.metamorphosis.client.metaclientconfig;import Com.taobao.metamorphosis.client.metamessagesessionfactory;import Com.taobao.metamorphosis.client.producer.messageproducer;import Com.taobao.metamorphosis.client.producer.sendresult;import Com.taobao.metamorphosis.exception.metaclientexception;import Com.taobao.metamorphosis.utils.zkutils;import Org.apache.flume.*;import Org.apache.flume.conf.configurable;import Org.apache.flume.sink.abstractsink;import Org.slf4j.logger;import Org.slf4j.loggerfactory;import Java.util.concurrent.countdownlatch;import Java.util.concurrent.executorservice;import Java.util.concurrent.executors;import java.util.concurrent.atomic.atomicinteger;/** * Function Description: * <p/> * This class is mainly to send flume collected data to Metaq message Queue * <p/> *-- --------------------------* Name: Mao Xiang * e-mail: [email pRotected] * Website: www.maoxiangyi.cn */public class Metaqsink extends Abstractsink implements configurable {private static Final Logger Logger = Loggerfactory.getlogger (Metaqsink.class); Private Messagesessionfactory sessionfactory; Private MessageProducer producer; Private String Zkconnect; Private String Zkroot; Private String topic; private int batchsize; private int threadnum; Private Executorservice executor; Public Metaqsink () {} @Override public void Configure (context context) {This.zkconnect = Context.getstri Ng ("Sink.zkconnect"); This.zkroot = context.getstring ("Sink.zkroot"); This.topic = context.getstring ("Sink.topic"); This.batchsize = Context.getinteger ("Sink.batchsize", 10000); This.threadnum = Context.getinteger ("Sink.threadnum", 50); Executor = Executors.newcachedthreadpool (); Metaclientconfig metaclientconfig = new Metaclientconfig (); Zkutils.zkconfig zkconfig = new ZKUTILS.ZKCOnfig (); Zkconfig.zkconnect = Zkconnect; Zkconfig.zkroot = Zkroot; Metaclientconfig.setzkconfig (Zkconfig); try {sessionfactory = new metamessagesessionfactory (metaclientconfig); } catch (Metaclientexception e) {e.printstacktrace (); Logger.error ("", e); throw new RuntimeException ("init error"); } producer = Sessionfactory.createproducer (); Logger.info ("Zkconnect:" + Zkconnect + ", Zkroot:" + zkroot + ", topic:" + topic "); } @Override Public Status process () throws Eventdeliveryexception {Long start = System.currenttimemillis (); Producer.publish (topic); Status result = Status.ready; Final Channel channel = Getchannel (); Final Atomicinteger al = new Atomicinteger (0); Final Countdownlatch CDL = new Countdownlatch (threadnum); for (int t = 0; t < threadnum; t++) {Executor.execute (new Runnable () { @Override public void Run () {event event = null; Transaction Transaction = null; int i = 0; try {transaction = Channel.gettransaction (); Transaction.begin (); Boolean starttransaction = false; for (i = 0; i < batchsize; i++) {event = Channel.take (); if (event! = null) {if (i = = 0) {Producer.begintransac tion (); StartTransaction = true; } final Sendresult Sendresult = producer. Sendmessag E (New Message (topic, event. GetBody ())); Check result if (!sendresult.issuccess ()) {Logger.error ("Send Message Failed,error message: "+ sendresult.geterrormessage ()); throw new RuntimeException ("Send message Failed,error message:" + Sendresult. GetErrorMessage ()); } else {logger.debug ("Send message successfully,sent to "+ sendresult.getpartition ()); }} else {//No event found, request Back-off semantics From the sink//runner//Result = Status.backoff; Break }} if (StartTransaction) {producer.commit (); } al.addandget (i); Transaction.commit (); } catch (Exception ex) {Logger.error ("error while rollback:", ex); try {producer.rollback (); } catch (Exception e) {e.printstacktrace (); } transaction.rollback (); } finally {Cdl.countdown (); Transaction.close (); } } }); } try {cdl.await (); } catch (Interruptedexception e) {e.printstacktrace (); } if (al.get () = = 0) {result = Status.backoff; } LogGer.info ("metaqsink_new,process:{},time:{},queue_size:{}", new object[] {al.get (), System.currenttimemilli S ()-start}); return result; }}
Flume Learning Installation