golang非同步kafka生產者

這是一個建立於 的文章，其中的資訊可能已經有所發展或是發生改變。

在實際業務情境中，為了提高系統的即時性，減輕日誌儲存壓力，需要將日誌直接生產至訊息中介軟體，減少flume或flumted收集所導致的延時及效能壓力，本文實現了一下功能：
實現了一個靜態調用的非同步生產者    AsyncProducer
封裝了一個用於非同步發送的生產器    Agent

 

//@descriptionkafka代理//@author chenbintao//@data2017-09-2710:30初稿//2017-09-2711:15規範代碼//2017-09-2814:15對發送邏輯進行了最佳化package kafkaAgentimport ("fmt""log""runtime/debug""strings""time""github.com/Shopify/sarama")const (_BROKER_LIST_ = `localhost:9092`)const (_LABEL_ = "[_kafkaAgent_]")var (IS_DEBUG = false_PAUSE_  = false)func SetDebug(debug bool) {IS_DEBUG = debug}type Agent struct {flag           boolBrokerList     stringTopicList      stringSendTimeOut    time.DurationReceiveTimeOut time.DurationAsyncProducer  sarama.AsyncProducer}func (this *Agent) Set(BrokerList, TopicList string, SendTimeOut, ReceiveTimeOut time.Duration) bool {//只允許初始化一次if this.flag {return false}this.flag = truethis.BrokerList = BrokerListthis.TopicList = TopicListthis.SendTimeOut = SendTimeOutthis.ReceiveTimeOut = ReceiveTimeOutthis.AsyncProducer = getProducer(this.BrokerList, this.SendTimeOut, true)if nil == this.AsyncProducer {return false}return this.Check()}func (this *Agent) Check() bool {if "" == this.BrokerList || "" == this.TopicList {return false}if 0 == this.SendTimeOut && 0 == this.ReceiveTimeOut {return false}return true}func (this *Agent) Send(msg string) bool {defer func() {if e, ok := recover().(error); ok {log.Println(_LABEL_, "WARN: panic in %v", e)log.Println(_LABEL_, string(debug.Stack()))this.AsyncProducer.Close()this.AsyncProducer = getProducer(this.BrokerList, this.SendTimeOut, true)}}()if !this.Check() {return false}return asyncProducer(this.AsyncProducer,this.TopicList,msg,)}//=========================================================================// asyncProducer 非同步生產者func AsyncProducer(kafka_list, topics, s string, timeout time.Duration) bool {if "" == kafka_list || "" == topics {return false}producer := getProducer(kafka_list, timeout, false)if nil == producer {return false}defer producer.Close()go func(p sarama.AsyncProducer) {errors := p.Errors()success := p.Successes()for {select {case err := <-errors:if err != nil {if IS_DEBUG {log.Println(_LABEL_, err)}return} else {return}case <-success:return}}}(producer)return asyncProducer(producer, topics, s)}func asyncProducer(p sarama.AsyncProducer, topics, s string) bool {if nil == p {return false}msg := &sarama.ProducerMessage{Topic: topics,Value: sarama.ByteEncoder(s),}p.Input() <- msgif IS_DEBUG {fmt.Println(_LABEL_, msg)}return true}func getProducer(kafka_list string, timeout time.Duration, monitor bool) sarama.AsyncProducer {config := sarama.NewConfig()config.Producer.Return.Successes = trueconfig.Producer.Timeout = timeoutproducer, err := sarama.NewAsyncProducer(strings.Split(kafka_list, ","), config)if err != nil {if IS_DEBUG {log.Println(_LABEL_, err)}}if monitor {//消費狀態訊息,防止死結go func(producer sarama.AsyncProducer) {if nil == producer {log.Println(_LABEL_, "getProducer() producer error!")return}errors := producer.Errors()success := producer.Successes()for {select {case err := <-errors:if err != nil {if IS_DEBUG {log.Println(_LABEL_, err)}continue} else {continue}case <-success:continue}}}(producer)}return producer}

 

713 次點擊