Custom Sink-kafka for Flume

1. Create a Agent,sink type to be specified as a custom sink
Vi/usr/local/flume/conf/agent3.conf
Agent3.sources=as1
Agent3.channels=c1
Agent3.sinks=s1

Agent3.sources.as1.type=avro
agent3.sources.as1.bind=0.0.0.0
agent3.sources.as1.port=41414
Agent3.sources.as1.channels=c1

Agent3.channels.c1.type=memory

Agent3.sinks.s1.type=storm.test.kafka.testkafkasink
Agent3.sinks.s1.channel=c1
2. Create custom Kafka Sink (custom Kafka sink packaging is the producer of Kafka), the code is as follows
Refer to Flume official development documentation: Http://flume.apache.org/FlumeDeveloperGuide.html#sink
Custom Kafkasink need to inherit Abstractsink class implementation configurable interface
The Kafka topic (test111) used in the sink must exist
Package Storm.test.kafka;

Import java.util.Properties;

Import Kafka.javaapi.producer.Producer;
Import Kafka.producer.KeyedMessage;
Import Kafka.producer.ProducerConfig;
Import Kafka.serializer.StringEncoder;

Import Org.apache.flume.Channel;
Import Org.apache.flume.Context;
Import org.apache.flume.Event;
Import org.apache.flume.EventDeliveryException;
Import org.apache.flume.Transaction;
Import org.apache.flume.conf.Configurable;
Import Org.apache.flume.sink.AbstractSink;

public class Testkafkasink extends Abstractsink implements configurable {

Producer<string, string> Producer;
String topic = "test111";

@Override
Public Status process () throws Eventdeliveryexception {
Status status = NULL;
Channel channel = Getchannel ();
Transaction Transaction = Channel.gettransaction ();
Transaction.begin ();
try {
Event event = Channel.take ();
if (event==null) {
Transaction.rollback ();
status = Status.backoff;
return status;
}
byte[] BODY = event.getbody ();
Final String msg = new string (body);
Final keyedmessage<string, string> message = new keyedmessage<string, string> (topic, MSG);
Producer.send (message);
Transaction.commit ();
status = Status.ready;
} catch (Exception e) {
Transaction.rollback ();
status = Status.backoff;
} finally {
Transaction.close ();
}

return status;
}

@Override
public void Configure (Context arg0) {
Properties prop = new properties ();
Prop.put ("Zookeeper.connect", "h5:2181,h6:2181,h7:2181");
Prop.put ("Metadata.broker.list", "h5:9092,h6:9092,h7:9092");
Prop.put ("Serializer.class", StringEncoder.class.getName ());
Producer = new producer<string, string> (new Producerconfig (prop));
}

}
After you package the code as Kafkasink.jar, copy it to the Flume/lib directory on the node where Flume resides, and then you need to Kafka_2.10-0.8.2.0.jar, Kafka-clients-0.8.2.0.jar, Metrics-core-2.2.0.jar, Scala-library-2.10.4.jar These 4 jar packages are copied to the Flume/lib directory on the node where Flume resides.
3. Start the agent of Flume custom Kafkasink
[Email protected] ~]# cd/usr/local/flume/
[[email protected] flume]# bin/flume-ng agent--conf conf/--conf-file conf/agent3.conf--name Agent3-dflume.root.logger =info,console
4, write the log to flume agent, the code is as follows
Log4j.properties
Log4j.rootlogger=info,flume
Log4j.appender.flume = Org.apache.flume.clients.log4jappender.Log4jAppender
Log4j.appender.flume.Hostname = 192.168.1.35
Log4j.appender.flume.Port = 41414
Log4j.appender.flume.UnsafeMode = True
Write the log to flume with the following code
Package com.mengyao.flume;

Import Java.io.File;
Import java.io.IOException;
Import java.util.Collection;
Import java.util.List;

Import Org.apache.commons.io.FileUtils;
Import Org.apache.log4j.Logger;

public class Flumeproducer {

private static list<string> Getlines () {
List<string> lines = null;
try {
Final collection<file> listfiles = fileutils.listfiles (New File ("d:/"), null, FALSE);
for (File file:listfiles) {
Lines = fileutils.readlines (file);
Break
}
} catch (IOException e) {
E.printstacktrace ();
}

return lines;
}

public static void Main (string[] args) throws Exception {
Final list<string> lines = Getlines ();
Final Logger Logger = Logger.getlogger (Flumeproducer.class);
for (String line:lines) {
Logger.info (line+ "\ T" +system.currenttimemillis ());
Thread.Sleep (1000);
}
}
}
Must join Flume-ng-log4jappender-1.5.0-cdh5.1.3-jar-with-dependencies.jar this dependent jar
5, using Kafka consumer consumption flume (custom Kafka sink used by the producers of Kafka) production data
1. Consumer Shell Code
[Email protected] kafka]# bin/kafka-console-consumer.sh--zookeeper h7:2181--topic test111--from-beginning # #kaf Ka cluster is h5, H6, H7;zookeeper cluster is H5, H6, H7. Using consumers on any Kafka node is the same

2. Consumer Java code
Package Storm.test.kafka;

Import Java.util.HashMap;
Import java.util.List;
Import Java.util.Map;
Import java.util.Properties;

Import Kafka.consumer.Consumer;
Import Kafka.consumer.ConsumerConfig;
Import Kafka.consumer.ConsumerIterator;
Import Kafka.consumer.KafkaStream;
Import Kafka.javaapi.consumer.ConsumerConnector;
Import Kafka.serializer.StringEncoder;

public class Testconsumer {

Static final String topic = "test111";

public static void Main (string[] args) {
Properties prop = new properties ();
Prop.put ("Zookeeper.connect", "h5:2181,h6:2181,h7:2181");
Prop.put ("Serializer.class", StringEncoder.class.getName ());
Prop.put ("Metadata.broker.list", "h5:9092,h6:9092,h7:9092");
Prop.put ("Group.id", "group1");
Consumerconnector consumer = consumer.createjavaconsumerconnector (new Consumerconfig (prop));
map<string, integer> topiccountmap = new hashmap<string, integer> ();
Topiccountmap.put (topic, 1);
Map<string, list<kafkastream<byte[], byte[]>>> messagestreams = Consumer.createmessagestreams ( TOPICCOUNTMAP);
Final kafkastream<byte[], byte[]> kafkastream = messagestreams.get (topic). get (0);
Consumeriterator<byte[], byte[]> iterator = Kafkastream.iterator ();
while (Iterator.hasnext ()) {
String msg = new String (Iterator.next (). message ());
System.out.println ("received message:" +msg);
}
}

}

Custom Sink-kafka for Flume