k8s與日誌--採用golang實現Fluent Bit的output外掛程式

採用golang實現Fluent Bit的output外掛程式

前言

目前社區日誌採集和處理的組件不少，之前elk方案中的logstash，cncf社區中的fluentd，efk方案中的filebeat,以及大資料用到比較多的flume。而Fluent Bit是一款用c語言編寫的高效能的日誌收集組件，整個架構源於fluentd。官方比較資料如下：

	Fluentd	Fluent Bit
Scope	Containers / Servers	Containers / Servers
Language	C & Ruby	C
Memory	~40MB	~450KB
Performance	High Performance	High Performance
Dependencies	Built as a Ruby Gem, it requires a certain number of gems.	Zero dependencies, unless some special plugin requires them.
Plugins	More than 650 plugins available	Around 35 plugins available
License	Apache License v2.0	Apache License v2.0

通過資料可以看出，fluent bit 佔用資源更少。適合採用fluent bit + fluentd 的方案，實現日誌中心化收集的方案。fluent bit主要負責採集，fluentd負責處理和傳送。

擴充output外掛程式

fluent bit 本身是C語言編寫，擴充外掛程式有一定的難度。可能官方考慮到這一點，實現了fluent-bit-go,可以實現採用go語言來編寫外掛程式，目前只支援output的編寫。
fluent-bit-go其實就是利用cgo,封裝了c介面。代碼比較簡單，主要分析其中一個關鍵檔案

package output/*#include <stdlib.h>#include "flb_plugin.h"#include "flb_output.h"*/import "C"import "fmt"import "unsafe"// Define constants matching Fluent Bit coreconst FLB_ERROR               =  C.FLB_ERRORconst FLB_OK                  =  C.FLB_OKconst FLB_RETRY               =  C.FLB_RETRYconst FLB_PROXY_OUTPUT_PLUGIN =  C.FLB_PROXY_OUTPUT_PLUGINconst FLB_PROXY_GOLANG        =  C.FLB_PROXY_GOLANG// Local type to define a plugin definitiontype FLBPlugin C.struct_flb_plugin_proxytype FLBOutPlugin C.struct_flbgo_output_plugin// When the FLBPluginInit is triggered by Fluent Bit, a plugin context// is passed and the next step is to invoke this FLBPluginRegister() function// to fill the required information: type, proxy type, flags name and// description.func FLBPluginRegister(ctx unsafe.Pointer, name string, desc string) int {    p := (*FLBPlugin) (unsafe.Pointer(ctx))    p._type = FLB_PROXY_OUTPUT_PLUGIN    p.proxy = FLB_PROXY_GOLANG    p.flags = 0    p.name  = C.CString(name)    p.description = C.CString(desc)    return 0}// Release resources allocated by the plugin initializationfunc FLBPluginUnregister(ctx unsafe.Pointer) {    p := (*FLBPlugin) (unsafe.Pointer(ctx))    fmt.Printf("[flbgo] unregistering %v\n", p)    C.free(unsafe.Pointer(p.name))    C.free(unsafe.Pointer(p.description))}func FLBPluginConfigKey(ctx unsafe.Pointer, key string) string {    _key := C.CString(key)    return C.GoString(C.output_get_property(_key, unsafe.Pointer(ctx)))}

主要是定義了一些編寫外掛程式需要用到的變數和方法，例如FLBPluginRegister註冊組件，FLBPluginConfigKey擷取設定檔設定參數等。
PS
實際上用golang調用fluent-bit-go，再加一些實際的商務邏輯實現，最終編譯成一個c-share的.so動態連結程式庫。

定製fluent-bit-kafka-ouput外掛程式

實際上，fluent-bit v0.13版本以後就提供了kafka output的外掛程式，但是實際項目中，並不滿足我們的需求，必須定製化。
當然接下來的代碼主要是作為一個demo,講清楚如何編寫一個output外掛程式。

代碼編寫和分析

先上代碼：

package mainimport (    "C"    "fmt"    "io"    "log"    "reflect"    "strconv"    "strings"    "time"    "unsafe"    "github.com/Shopify/sarama"    "github.com/fluent/fluent-bit-go/output"    "github.com/ugorji/go/codec")var (    brokers    []string    producer   sarama.SyncProducer    timeout    = 0 * time.Minute    topic      string    module     string    messageKey string)//export FLBPluginRegisterfunc FLBPluginRegister(ctx unsafe.Pointer) int {    return output.FLBPluginRegister(ctx, "out_kafka", "Kafka Output Plugin.!")}//export FLBPluginInit// ctx (context) pointer to fluentbit context (state/ c code)func FLBPluginInit(ctx unsafe.Pointer) int {    if bs := output.FLBPluginConfigKey(ctx, "brokers"); bs != "" {        brokers = strings.Split(bs, ",")    } else {        log.Printf("you must set brokers")        return output.FLB_ERROR    }    if tp := output.FLBPluginConfigKey(ctx, "topics"); tp != "" {        topic = tp    } else {        log.Printf("you must set topics")        return output.FLB_ERROR    }    if mo := output.FLBPluginConfigKey(ctx, "module"); mo != "" {        module = mo    } else {        log.Printf("you must set module")        return output.FLB_ERROR    }    if key := output.FLBPluginConfigKey(ctx, "message_key"); key != "" {        messageKey = key    } else {        log.Printf("you must set message_key")        return output.FLB_ERROR    }    config := sarama.NewConfig()    config.Producer.Return.Successes = true    if required_acks := output.FLBPluginConfigKey(ctx, "required_acks"); required_acks != "" {        if acks, err := strconv.Atoi(required_acks); err == nil {            config.Producer.RequiredAcks = sarama.RequiredAcks(acks)        }    }    if compression_codec := output.FLBPluginConfigKey(ctx, "compression_codec"); compression_codec != "" {        if codec, err := strconv.Atoi(compression_codec); err == nil {            config.Producer.Compression = sarama.CompressionCodec(codec)        }    }    if max_retry := output.FLBPluginConfigKey(ctx, "max_retry"); max_retry != "" {        if max_retry, err := strconv.Atoi(max_retry); err == nil {            config.Producer.Retry.Max = max_retry        }    }    if timeout == 0 {        timeout = 5 * time.Minute    }    // If Kafka is not running on init, wait to connect    deadline := time.Now().Add(timeout)    for tries := 0; time.Now().Before(deadline); tries++ {        var err error        if producer == nil {            producer, err = sarama.NewSyncProducer(brokers, config)        }        if err == nil {            return output.FLB_OK        }        log.Printf("Cannot connect to Kafka: (%s) retrying...", err)        time.Sleep(time.Second * 30)    }    log.Printf("Kafka failed to respond after %s", timeout)    return output.FLB_ERROR}//export FLBPluginFlush// FLBPluginFlush is called from fluent-bit when data need to be sent. is called from fluent-bit when data need to be sent.func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int {    var h codec.MsgpackHandle    var b []byte    var m interface{}    var err error    b = C.GoBytes(data, length)    dec := codec.NewDecoderBytes(b, &h)    // Iterate the original MessagePack array    var msgs []*sarama.ProducerMessage    for {        // decode the msgpack data        err = dec.Decode(&m)        if err != nil {            if err == io.EOF {                break            }            log.Printf("Failed to decode msgpack data: %v\n", err)            return output.FLB_ERROR        }        // Get a slice and their two entries: timestamp and map        slice := reflect.ValueOf(m)        data := slice.Index(1)        // Convert slice data to a real map and iterate        mapData := data.Interface().(map[interface{}]interface{})        flattenData, err := Flatten(mapData, "", UnderscoreStyle)        if err != nil {            break        }        message := ""        host := ""        for k, v := range flattenData {            value := ""            switch t := v.(type) {            case string:                value = t            case []byte:                value = string(t)            default:                value = fmt.Sprintf("%v", v)            }            if k == "pod_name" {                host = value            }            if k == messageKey {                message = value            }        }        if message == "" || host == "" {            break        }        m := &sarama.ProducerMessage{            Topic: topic,            Key:   sarama.StringEncoder(fmt.Sprintf("host=%s|module=%s", host, module)),            Value: sarama.ByteEncoder(message),        }        msgs = append(msgs, m)    }    err = producer.SendMessages(msgs)    if err != nil {        log.Printf("FAILED to send kafka message: %s\n", err)        return output.FLB_ERROR    }    return output.FLB_OK}//export FLBPluginExitfunc FLBPluginExit() int {    producer.Close()    return output.FLB_OK}func main() {}

• FLBPluginExit 外掛程式退出的時候需要執行的一些方法，比如關閉串連。
• FLBPluginRegister 註冊外掛程式
• FLBPluginInit 外掛程式初始化
• FLBPluginFlush flush到資料到output
• FLBPluginConfigKey 擷取設定檔中參數

PS
當然除了FLBPluginConfigKey之外，也可以通過擷取環境變數來獲得設定參數。
ctx相當於一個上下文，負責之間的資料的傳遞。

編譯和執行

編譯的時候

go build -buildmode=c-shared -o out_kafka.so .

產生out_kafka.so

執行的時候

/fluent-bit/bin/fluent-bit" -c /fluent-bit/etc/fluent-bit.conf -e /fluent-bit/out_kafka.so

總結

採用類似的編寫結構，就可以定製化自己的輸出外掛程式了。