Application of high-throughput distributed subscription message system Kafka--spring-integration-kafka

I. Overview

The spring integration Kafka is based on the Apache Kafka and spring integration to integrate KAFKA, which facilitates development configuration.

Second, the configuration

1, Spring-kafka-consumer.xml

<?xml version= "1.0" encoding= "UTF-8"? ><beans xmlns= "Http://www.springframework.org/schema/beans" xmlns: Xsi= "Http://www.w3.org/2001/XMLSchema-instance" xmlns:int= "Http://www.springframework.org/schema/integration" Xmlns:int-kafka= "Http://www.springframework.org/schema/integration/kafka" xmlns:task= "http// Www.springframework.org/schema/task "xsi:schemalocation=" http://www.springframework.org/schema/integration/                        Kafka Http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd Http://www.springframework.org/schema/integration http://www.springframework.org/s                         Chema/integration/spring-integration.xsd Http://www.springframework.org/schema/beans Http://www.springframework.org/schema/beans/spring-beans.xsd HTTP://WWW.SPRINGFRAMEWORK.O Rg/schema/task Http://www.springframework.org/schema/task/spring-task.XSD "><!--topic Test conf--><int:channel id=" Inputfromkafka "><int:dispatcher task-executor=" Kafkamessageexecutor "/></int:channel> <!--zookeeper configuration can be configured with multiple--><int-kafka:zookeeper-connect id= "Zookeeperconnect" zk-connect= "192.168.1.237:2181" zk-connection-timeout= "6000" zk-session-timeout= "6000" Zk-sync-time= ""/> <!--Channel configuration auto-startup= "true" otherwise receive no data--><int-kafka: Inbound-channel-adapterid= "Kafkainboundchanneladapter" kafka-consumer-context-ref= "ConsumerContext" Auto-startup = "true" channel= "Inputfromkafka" ><int:poller fixed-delay= "1" time-unit= "MILLISECONDS"/></int-kafka: inbound-channel-adapter> <task:executor id= "Kafkamessageexecutor" pool-size= "8" keep-alive= "120" queue-capacity= "/><bean id=" Kafkadecoder "class=" Org.springframework.integration.kafka.serializer.common.StringDecoder "/><bean id=" Consumerproperties " class= "Org.springframework.beans.factory.config.PropertiesFactoryBean" &GT;<property name= "Properties" ><props><prop key= "Auto.offset.reset" >smallest</prop>< Prop key= "Socket.receive.buffer.bytes" >10485760</prop> <!--10M--><prop key= " Fetch.message.max.bytes ">5242880</prop><prop key=" auto.commit.interval.ms ">1000</prop>< /props></property></bean> <!--message received been--><bean id= "Kafkaconsumerservice" class= " Com.sunney.service.impl.KafkaConsumerService "/> <!--Specifies the method to receive--><int:outbound-channel-adapter channel= "Inputfromkafka" ref= "Kafkaconsumerservice" method= "ProcessMessage"/><int-kafka:consumer-context id= " Consumercontext "consumer-timeout=" "zookeeper-connect=" Zookeeperconnect "consumer-properties=" Consumerproperties "><int-kafka:consumer-configurations><int-kafka:consumer-configurationgroup-id=" Default1 "value-decoder=" Kafkadecoder "key-decoder=" Kafkadecoder "max-messages=" ><!--two topic configurations-- <int-kafka:topic id= "MytopIC "streams=" 4 "/><int-kafka:topic id=" Sunneytopic "streams=" 4 "/></int-kafka:consumer-configuration     ></int-kafka:consumer-configurations></int-kafka:consumer-context></beans>

2, Spring-kafka-producer.xml

<?xml version= "1.0" encoding= "UTF-8"? ><beans xmlns= "Http://www.springframework.org/schema/beans" xmlns: Xsi= "Http://www.w3.org/2001/XMLSchema-instance" xmlns:int= "Http://www.springframework.org/schema/integration" Xmlns:int-kafka= "Http://www.springframework.org/schema/integration/kafka" xmlns:task= "http// Www.springframework.org/schema/task "xsi:schemalocation=" http://www.springframework.org/schema/integration/ Kafka Http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd/HTTP Www.springframework.org/schema/integration http://www.springframework.org/schema/integration/ Spring-integration.xsd Http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/ Spring-beans.xsd Http://www.springframework.org/schema/task http://www.springframework.org/schema/task/ Spring-task.xsd "> <!--Commons config--><bean id=" Stringserializer "class=" Org.apache.kafka.common.serialization.StringSerializer "/><beAn id= "Kafkaencoder" class= " Org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder "> < Constructor-arg value= "java.lang.String"/> </bean><bean id= "producerproperties" class= " Org.springframework.beans.factory.config.PropertiesFactoryBean "><property name=" Properties "><props ><prop key= "topic.metadata.refresh.interval.ms" >3600000</prop><prop key= " Message.send.max.retries ">5</prop><prop key=" Serializer.class ">kafka.serializer.stringencoder </prop><prop key= "Request.required.acks" >1</prop></props></property></bean> <!--topic Test config--><int:channel id= "kafkatopictest" ><int:queue/></int:channel>< Int-kafka:outbound-channel-adapterid= "Kafkaoutboundchanneladaptertopictest" kafka-producer-context-ref= " Producercontexttopictest "auto-startup=" true "channel=" Kafkatopictest "order=" 3 "><int:poller fixed-delay=" "Time-unit= "MILLISECONDS" receive-timeout= "1" task-executor= "Taskexecutor"/></int-kafka:outbound-channel-adapter> <task:executor id= "Taskexecutor" pool-size= "5" keep-alive= "+" queue-capacity= "/><!--<bean id=" Kafkaencoder "class=" Org.springframework.integration.kafka.serializer.avro.AvroSpecificDatumBackedKafkaEncoder " ><constructor-arg value= "Com.company.AvroGeneratedSpecificRecord"/></bean>--><int-kafka: Producer-context id= "Producercontexttopictest" producer-properties= "Producerproperties" ><int-kafka: Producer-configurations> <!--multiple topic configuration--><int-kafka:producer-configurationbroker-list= " 192.168.1.237:9090,192.168.1.237:9091,192.168.1.237:9092 "key-serializer=" Stringserializer "Value-class-type = "Java.lang.String" value-serializer= "Stringserializer" topic= "Mytopic"/><int-kafka:producer-configur Ationbroker-list= "192.168.1.237:9090,192.168.1.237:9091,192.168.1.237:9092" key-serializer= "StringSeriaLizer "value-class-type=" java.lang.String "value-serializer=" Stringserializer "topic=" Sunneyt OPIc "/></int-kafka:producer-configurations></int-kafka:producer-context></beans>

3, Send Message interface Kafkaservice

Package com.sunney.service;/** * Class Kafkaservice.java Implementation description: Send Message interface class * @author Sunney April 30, 2016 morning 11:30:53 */public Interface Kafkaservice {    /**     * Send Message     * @param topic subject     * @param obj Send content */public    void Senduserinf O (String topic, Object obj);}

4, Send Message implementation class Kafkaserviceimpl

Package Com.sunney.service.impl;import Org.springframework.beans.factory.annotation.autowired;import Org.springframework.beans.factory.annotation.qualifier;import Org.springframework.integration.kafka.support.kafkaheaders;import Org.springframework.integration.support.messagebuilder;import Org.springframework.messaging.MessageChannel; Import Org.springframework.stereotype.service;import com.sunney.service.kafkaservice;/** * Implementation description of Class Kafkaserviceimpl.java: Send Message Implementation class  * @author Sunney April 30, 2016 morning 11:31:13 */@Service ("Kafkaservice") public class Kafkaserviceimpl  implements kafkaservice{    @Autowired    @Qualifier ("Kafkatopictest")    Messagechannel Channel;    public void Senduserinfo (String topic, Object obj) {        channel.send (messagebuilder.withpayload (obj))                                    . SetHeader (Kafkaheaders.topic,topic)                                    . Build ());}    }

5. Consumer receiving class Kafkaconsumerservice

Package Com.sunney.service.impl;import Java.util.collection;import Java.util.iterator;import Java.util.linkedhashmap;import java.util.list;import java.util.map;import Java.util.set;import Org.slf4j.Logger; Import Org.slf4j.loggerfactory;import com.alibaba.fastjson.json;import com.sunney.service.userdto;/** * Class Kafkaconsumerservice.java implementation Description: Consumer Receive class * * @author Sunney April 30, 2016 morning 11:46:14 */public class Kafkaconsumerservice {s    Tatic final Logger Logger = Loggerfactory.getlogger (Kafkaconsumerservice.class); public void ProcessMessage (map<string, Map<integer, string>> msgs) {logger.info ("===============proces        smessage=============== "); For (map.entry<string, Map<integer, string>> Entry:msgs.entrySet ()) {Logger.info ("============to            Pic: "+ entry.getkey ());            Linkedhashmap<integer, string> messages = (Linkedhashmap<integer, string>) Entry.getvalue ();           set<integer> keys = Messages.keyset (); for (Integer I:keys) logger.info ("======partition:" + i);            collection<string> values = Messages.values (); for (iterator<string> Iterator = Values.iterator (); Iterator.hasnext ();)                {String message = "[" +iterator.next () + "]";                Logger.info ("=====message:" + message);                  List<userdto> userlist = Json.parsearray (message, userdto.class);            Logger.info ("=====userlist.size:" + userlist.size ()); }        }    }}

6, Pom

<dependencies><dependency><groupid>org.springframework.boot</groupid><artifactid >spring-boot-starter-web</artifactId></dependency><dependency><groupId> Org.springframework.integration</groupid><artifactid>spring-integration-kafka</artifactid> <version>1.3.0.release</version></dependency><dependency><groupid>junit</ groupid><artifactid>junit</artifactid><version>4.11</version><scope>test</ Scope></dependency><dependency><groupid> org.apache.avro</groupid><artifactid> Avro</artifactid><version>1.7.7</version></dependency><dependency><groupid >com.alibaba</groupid><artifactid>fastjson</artifactid><version>1.2.7</version ></dependency></dependencies>

Six, source code address: Https://github.com/sunney2010/kafka-demo

Vii. problems encountered

1. The consumer port does not receive the message

Spring-kafka-consumer.xml Auto-startup set to True

Application of high-throughput distributed subscription message system Kafka--spring-integration-kafka