Golang Asynchronous Kafka producer

This is a creation in Article, where the information may have evolved or changed.

In the actual business scenario, in order to improve the real-time performance of the system, reduce the pressure of log storage, the need to direct the production of logs to the message middleware, reduce the flume or flumted collection caused by the delay and pressure, this paper realizes the function:
Implements a static call to the asynchronous producer Asyncproducer
Encapsulates a producer Agent for asynchronous sending

@descriptionkafka Agent//@author chenbintao//@data2017 -09-2710:30 First draft//2017-09-2711:15 specification code// 2017-09-2814:15 optimized the Send logic package kafkaagentimport ("FMT" "Log" "Runtime/debug" "Strings" "Time" github.com/shopify/ Sarama ") const (_broker_list_ = ' localhost:9092 ') const (_LABEL_ =" [_kafkaagent_] ") var (is_debug = False_pause_ = False) Fu NC setdebug (debug bool) {is_debug = Debug}type Agent struct {flag boolbrokerlist stringtopiclist string Sendtimeout time. Durationreceivetimeout time. Durationasyncproducer Sarama. Asyncproducer}func (This *agent) Set (brokerlist, topiclist string, Sendtimeout, ReceiveTimeout time. Duration) bool {//Only allows initialization once if This.flag {return false}this.flag = Truethis. Brokerlist = Brokerlistthis.topiclist = Topiclistthis.sendtimeout = Sendtimeoutthis.receivetimeout = Receivetimeoutthis.asyncproducer = Getproducer (this. Brokerlist, this. Sendtimeout, true) if nil = = this. Asyncproducer {return False}return this. Check ()}func (this *agent) check () bool {if "" = = this. Brokerlist | | "" = = this. topiclist {return False}if 0 = = this. Sendtimeout && 0 = = this. ReceiveTimeout {return False}return true}func (this *agent) Send (msg string) bool {defer func () {if e, ok: = Recover (). ( Error); OK {log. Println (_label_, "Warn:panic in%v", e) log. Println (_label_, String (Debug. Stack ())) this. Asyncproducer.close () this. Asyncproducer = Getproducer (this. Brokerlist, this. Sendtimeout, True)}} () if!this. Check () {return False}return asyncproducer (this. Asyncproducer,this. Topiclist,msg,)}//=========================================================================//AsyncProducer Asynchronous producer Func Asyncproducer (kafka_list, topics, s string, timeout time. Duration) bool {if "" = = Kafka_list | | "" = = topics {return False}producer: = Getproducer (Kafka_list, timeout, false) if NI L = = Producer {return False}defer producer. Close () go func (P Sarama. Asyncproducer) {Errors: = P.errors () Success: = P.successes () for {select {case ERR: = <-errors:if Err! = Nil {if Is_debu G {log. Println (_label_, err)}return} else {Return}case <-success:return}}} (producer) return Asyncproducer (producer, topics, s)}func Asyncproducer (P Sarama. Asyncproducer, topics, s string) bool {if nil = = p {return false}msg: = &sarama. Producermessage{topic:topics,value:sarama. Byteencoder (s),}p.input () <-msgif is_debug {fmt. Println (_label_, msg)}return true}func getproducer (kafka_list string, timeout time. Duration, monitor bool) Sarama. asyncproducer {config: = Sarama. Newconfig () config. Producer.Return.Successes = Trueconfig. Producer.timeout = Timeoutproducer, err: = Sarama. Newasyncproducer (Strings. Split (Kafka_list, ","), config) if err! = Nil {if Is_debug {log. Println (_label_, err)}}if Monitor {//consume status messages to prevent deadlock go func (producer Sarama. Asyncproducer) {if nil = = producer {log. Println (_label_, "Getproducer () producer error!") Return}errors: = producer. Errors () Success: = producer. Successes () for {select {case ERR: = <-errors:if Err! = Nil {if Is_debug {log. Println (_label_, err)}continue} else {continue}case <-success:continue}}} (producer)}return producer} 

713 Reads