漫遊kafka實戰篇之搭建Kafka開發環境

標籤：kafka

轉載註明出處：http://blog.csdn.net/honglei915/article/details/37563647上篇文章中我們搭建了kafka的伺服器，並可以使用Kafka的命令列工具建立topic，發送和接收訊息。下面我們來搭建kafka的開發環境。
添加依賴

搭建開發環境需要引入kafka的jar包，一種方式是將Kafka安裝包中lib下的jar包加入到項目的classpath中，這種比較簡單了。不過我們使用另一種更加流行的方式：使用maven管理jar包依賴。建立好maven項目後，在pom.xml中添加以下依賴：               

<dependency>         <groupId> org.apache.kafka</groupId >         <artifactId> kafka_2.10</artifactId >         <version> 0.8.0</ version></dependency>

添加依賴後你會發現有兩個jar包的依賴找不到。沒關係我都幫你想好了，點擊這裡下載這兩個jar包，解壓後你有兩種選擇，第一種是使用mvn的install命令將jar包安裝到本地倉庫，另一種是直接將解壓後的檔案夾拷貝到mvn本地倉庫的com檔案夾下，比如我的本地倉庫是d:\mvn,完成後我的目錄結構是這樣的：


配置程式
首先是一個充當設定檔作用的介面,配置了Kafka的各種串連參數：

package com.sohu.kafkademon;public interface KafkaProperties{    final static String zkConnect = "10.22.10.139:2181";    final static String groupId = "group1";    final static String topic = "topic1";    final static String kafkaServerURL = "10.22.10.139";    final static int kafkaServerPort = 9092;    final static int kafkaProducerBufferSize = 64 * 1024;    final static int connectionTimeOut = 20000;    final static int reconnectInterval = 10000;    final static String topic2 = "topic2";    final static String topic3 = "topic3";    final static String clientId = "SimpleConsumerDemoClient";}

producer

package com.sohu.kafkademon;import java.util.Properties;import kafka.producer.KeyedMessage;import kafka.producer.ProducerConfig;/** * @author leicui [email protected] */public class KafkaProducer extends Thread{    private final kafka.javaapi.producer.Producer<Integer, String> producer;    private final String topic;    private final Properties props = new Properties();    public KafkaProducer(String topic)    {        props.put("serializer.class", "kafka.serializer.StringEncoder");        props.put("metadata.broker.list", "10.22.10.139:9092");        producer = new kafka.javaapi.producer.Producer<Integer, String>(new ProducerConfig(props));        this.topic = topic;    }    @Override    public void run() {        int messageNo = 1;        while (true)        {            String messageStr = new String("Message_" + messageNo);            System.out.println("Send:" + messageStr);            producer.send(new KeyedMessage<Integer, String>(topic, messageStr));            messageNo++;            try {                sleep(3000);            } catch (InterruptedException e) {                // TODO Auto-generated catch block                e.printStackTrace();            }        }    }}

consumer

package com.sohu.kafkademon;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Properties;import kafka.consumer.ConsumerConfig;import kafka.consumer.ConsumerIterator;import kafka.consumer.KafkaStream;import kafka.javaapi.consumer.ConsumerConnector;/** * @author leicui [email protected] */public class KafkaConsumer extends Thread{    private final ConsumerConnector consumer;    private final String topic;    public KafkaConsumer(String topic)    {        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(                createConsumerConfig());        this.topic = topic;    }    private static ConsumerConfig createConsumerConfig()    {        Properties props = new Properties();        props.put("zookeeper.connect", KafkaProperties.zkConnect);        props.put("group.id", KafkaProperties.groupId);        props.put("zookeeper.session.timeout.ms", "40000");        props.put("zookeeper.sync.time.ms", "200");        props.put("auto.commit.interval.ms", "1000");        return new ConsumerConfig(props);    }    @Override    public void run() {        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();        topicCountMap.put(topic, new Integer(1));        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);        KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);        ConsumerIterator<byte[], byte[]> it = stream.iterator();        while (it.hasNext()) {            System.out.println("receive：" + new String(it.next().message()));            try {                sleep(3000);            } catch (InterruptedException e) {                e.printStackTrace();            }        }    }}

簡單的發送接收
運行下面這個程式，就可以進行簡單的發送接收訊息了：

package com.sohu.kafkademon;/** * @author leicui [email protected] */public class KafkaConsumerProducerDemo{    public static void main(String[] args)    {        KafkaProducer producerThread = new KafkaProducer(KafkaProperties.topic);        producerThread.start();        KafkaConsumer consumerThread = new KafkaConsumer(KafkaProperties.topic);        consumerThread.start();    }}

進階別的consumer
下面是比較負載的發送接收的程式：

package com.sohu.kafkademon;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Properties;import kafka.consumer.ConsumerConfig;import kafka.consumer.ConsumerIterator;import kafka.consumer.KafkaStream;import kafka.javaapi.consumer.ConsumerConnector;/** * @author leicui [email protected] */public class KafkaConsumer extends Thread{    private final ConsumerConnector consumer;    private final String topic;    public KafkaConsumer(String topic)    {        consumer = kafka.consumer.Consumer.createJavaConsumerConnector(                createConsumerConfig());        this.topic = topic;    }    private static ConsumerConfig createConsumerConfig()    {        Properties props = new Properties();        props.put("zookeeper.connect", KafkaProperties.zkConnect);        props.put("group.id", KafkaProperties.groupId);        props.put("zookeeper.session.timeout.ms", "40000");        props.put("zookeeper.sync.time.ms", "200");        props.put("auto.commit.interval.ms", "1000");        return new ConsumerConfig(props);    }    @Override    public void run() {        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();        topicCountMap.put(topic, new Integer(1));        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);        KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);        ConsumerIterator<byte[], byte[]> it = stream.iterator();        while (it.hasNext()) {            System.out.println("receive：" + new String(it.next().message()));            try {                sleep(3000);            } catch (InterruptedException e) {                e.printStackTrace();            }        }    }}