Flume Learning Installation

The recent project team has the need to tap the stream log to collect, learn a bit flume and install successfully. The relevant information to record a bit.

1) Download flume1.5 version
wget http://www.apache.org/dyn/closer.cgi/flume/1.5.0.1/apache-flume-1.5.0.1-bin.tar.gz
2) Unzip the flume1.5
TAR-ZXVF apache-flume-1.5.0.1-bin.tar.gz
3) Configure Environment variables
JDK is installed
Export Flume_home=/xxx/xx/apache-flume-1.5.0.1-bin
Export path= $FLUME _home/bin: $PATH </p><p align= "left" >
4) Configure Conf related files </p><p align= "left" >
4.1) configuration flume-env.sh mainly set up a bit java_home
4.2) Configuration Log4j.properties
If the test environment is commented out Flume.root.logger=info,logfile Select Flume.root.logger=debug,console Print the log to the console
4.3) Configure flume-conf.properties This file name can be arbitrarily changed when you run the command, specify the properties file you created yourself
#set Agent name for A1 sources name for R1 sinks name for K1 channels name for C1
A1.sources = R1
A1.sinks = K1
A1.channels = C1
Sources component type executes Linux command for exec
A1.sources.r1.type = Exec
A1.sources.r1.command = Tail-f/home/hadoop/flume/flume/conf/source.txt (big difference in size tail-f solves a big problem for us)


Sinks Component Type is logger
A1.sinks.k1.type = Logger
Channels component type is memory
A1.channels.c1.type = Memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100


Connect the sources, the sinks and the pipes.
A1.sources.r1.channels = C1
A1.sinks.k1.channel = C1


5) Run the command in the Flume directory
Bin/flume-ng agent-n a1-f test/source-tail-sink-logger.properties--conf conf


A preliminary example is completed. At present, our production environment is two nodes to the Metaq inside the production of data. Metaq Customizing a sink (custom sink see following code)
Remember to put the Metaq related jar under Flume/lib Gecko-1.1.4.jar Metamorphosis-client-1.4.6.2.jar Metamorphosis-commons-1.4.6.2.jar Zkclient-0.3.jar Zookeeper-3.4.3.jar


A1.sources = R1
A1.sinks = K1
A1.channels = C1


A1.sources.r1.type = Exec
A1.sources.r1.command = Tail- F/home/hadoop/flume/flume/conf/source.txt
A1.sinks.k1.type = com. XX.flume.sink.MetaQSink
A1.sinks.k1.sink.zkConnect = 0.0.0.0:2181,0.0.0.0:2181,0.0.0.0:2181
A1.sinks.k1.sink.zkRoot =/meta (this directory must be written dead)
A1.sinks.k1.sink.topic = XXXX
A1.sinks.k1.sink.batchSize = 20000
#a1. Channels.c1.type = Memory
#a1. channels.c1.capacity = 1000000
#a1. channels.c1.transactionCapacity = 100000
A1.channels.c1.type = File
A1.channels.c1.checkpointDir =/home/hadoop/flume/flume/channel/checkpoint
A1.channels.c1.dataDirs =/home/hadoop/flume/flume/channel/data


A1.sources.r1.channels = C1
A1.sinks.k1.channel = C1

Custom Sink Code

</pre><pre name= "code" class= "java" >package com.jd.flume.sink;import com.taobao.metamorphosis.Message; Import Com.taobao.metamorphosis.client.messagesessionfactory;import Com.taobao.metamorphosis.client.metaclientconfig;import Com.taobao.metamorphosis.client.metamessagesessionfactory;import Com.taobao.metamorphosis.client.producer.messageproducer;import Com.taobao.metamorphosis.client.producer.sendresult;import Com.taobao.metamorphosis.exception.metaclientexception;import Com.taobao.metamorphosis.utils.zkutils;import Org.apache.flume.*;import Org.apache.flume.conf.configurable;import Org.apache.flume.sink.abstractsink;import Org.slf4j.logger;import Org.slf4j.loggerfactory;import Java.util.concurrent.countdownlatch;import Java.util.concurrent.executorservice;import Java.util.concurrent.executors;import java.util.concurrent.atomic.atomicinteger;/** * Function Description: * <p/> * This class is mainly to send flume collected data to Metaq message Queue * <p/> *-- --------------------------* Name: Mao Xiang * e-mail: [email pRotected] * Website: www.maoxiangyi.cn */public class Metaqsink extends Abstractsink implements configurable {private static    Final Logger Logger = Loggerfactory.getlogger (Metaqsink.class);    Private Messagesessionfactory sessionfactory;    Private MessageProducer producer;    Private String Zkconnect;    Private String Zkroot;    Private String topic;    private int batchsize;    private int threadnum;    Private Executorservice executor; Public Metaqsink () {} @Override public void Configure (context context) {This.zkconnect = Context.getstri        Ng ("Sink.zkconnect");        This.zkroot = context.getstring ("Sink.zkroot");        This.topic = context.getstring ("Sink.topic");        This.batchsize = Context.getinteger ("Sink.batchsize", 10000);        This.threadnum = Context.getinteger ("Sink.threadnum", 50);        Executor = Executors.newcachedthreadpool ();        Metaclientconfig metaclientconfig = new Metaclientconfig (); Zkutils.zkconfig zkconfig = new ZKUTILS.ZKCOnfig ();        Zkconfig.zkconnect = Zkconnect;        Zkconfig.zkroot = Zkroot;        Metaclientconfig.setzkconfig (Zkconfig);        try {sessionfactory = new metamessagesessionfactory (metaclientconfig);            } catch (Metaclientexception e) {e.printstacktrace ();            Logger.error ("", e);        throw new RuntimeException ("init error");        } producer = Sessionfactory.createproducer ();    Logger.info ("Zkconnect:" + Zkconnect + ", Zkroot:" + zkroot + ", topic:" + topic ");        } @Override Public Status process () throws Eventdeliveryexception {Long start = System.currenttimemillis ();        Producer.publish (topic);        Status result = Status.ready;        Final Channel channel = Getchannel ();        Final Atomicinteger al = new Atomicinteger (0);        Final Countdownlatch CDL = new Countdownlatch (threadnum);         for (int t = 0; t < threadnum; t++) {Executor.execute (new Runnable () {       @Override public void Run () {event event = null;                    Transaction Transaction = null;                    int i = 0;                        try {transaction = Channel.gettransaction ();                        Transaction.begin ();                        Boolean starttransaction = false;                            for (i = 0; i < batchsize; i++) {event = Channel.take (); if (event! = null) {if (i = = 0) {Producer.begintransac                                    tion ();                                StartTransaction = true; } final Sendresult Sendresult = producer. Sendmessag                                E (New Message (topic, event. GetBody ()));                        Check result        if (!sendresult.issuccess ()) {Logger.error ("Send Message Failed,error message:                                    "+ sendresult.geterrormessage ());                                                    throw new RuntimeException ("Send message Failed,error message:"                                + Sendresult. GetErrorMessage ());  } else {logger.debug ("Send message successfully,sent to                                "+ sendresult.getpartition ());                                }} else {//No event found, request Back-off semantics  From the sink//runner//Result                              = Status.backoff;  Break                        }} if (StartTransaction) {producer.commit ();                        } al.addandget (i);                    Transaction.commit ();                        } catch (Exception ex) {Logger.error ("error while rollback:", ex);                        try {producer.rollback ();                        } catch (Exception e) {e.printstacktrace ();                    } transaction.rollback ();                        } finally {Cdl.countdown ();                    Transaction.close ();        }                }            });        } try {cdl.await ();        } catch (Interruptedexception e) {e.printstacktrace ();        } if (al.get () = = 0) {result = Status.backoff; } LogGer.info ("metaqsink_new,process:{},time:{},queue_size:{}", new object[] {al.get (), System.currenttimemilli        S ()-start});    return result; }}

Flume Learning Installation