Flume Customizing the Deserializer Deserializer

Requirements background:

When using flume for log collection, the error message will print multiple lines of the stack, and multiple lines of information need to be merged into one line and packaged into an event for transmission.

Solution Ideas:　

Addressing these requirements can be achieved through custom interceptors and custom Deserializer. There are more information about custom interceptors on the web, but considering the location and usage scenarios of interceptors, interceptors should not be used for multiple event splitting combinations, and if Flume has concurrent processing, it is not guaranteed that the Read event is sequential. The lookup data found that the deserialization of the custom flume was more reasonable and secure.

Implementation steps:

1: Create a new class to implement the Eventdeserializer interface

2: Overriding the Readevent () method or the Readevents method

3: Modify the Flume configuration file to set the Sources.deserializer property to a custom class

Source:

1: Custom Deserializer---> Mylinedeserializer

/*** Licensed to the Apache software Foundation (ASF) under one * or more contributor license agreements.  See the NOTICE file * Distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * under the Apache License, Version 2.0 (The * "License");  You are not a use of this file except in compliance * with the License. Obtain a copy of the License at * <p> *http://www.apache.org/licenses/LICENSE-2.0* <p> * unless required by applicable law or agreed to in writing, software * Distributed under the License is Distributed on an ' as is ' BASIS, * without warranties or CONDITIONS of any KIND, either express OR implied. * See the License for the specific language governing permissions and * limitations under the License. */ PackageCom.xxx.flume.serializer;Importcom.google.common.collect.Lists;ImportOrg.apache.flume.Context;Importorg.apache.flume.Event;Importorg.apache.flume.annotations.InterfaceAudience;Importorg.apache.flume.annotations.InterfaceStability;ImportOrg.apache.flume.event.EventBuilder;ImportOrg.apache.flume.serialization.EventDeserializer;ImportOrg.apache.flume.serialization.ResettableInputStream;ImportOrg.slf4j.Logger;Importorg.slf4j.LoggerFactory;Importjava.io.IOException;ImportJava.nio.charset.Charset;Importjava.util.List;/*** A deserializer that parses the text lines from A file.*/@InterfaceAudience. private@interfacestability.evolving Public classMylinedeserializerImplementsEventdeserializer {Private Static FinalLogger Logger =Loggerfactory.getlogger (Mylinedeserializer.class); Private FinalResettableinputstream in; Private FinalCharset Outputcharset; Private Final intmaxlinelength; Private volatile BooleanIsOpen;  Public Static FinalString Out_charset_key = "Outputcharset";  Public Static FinalString Charset_dflt = "UTF-8";  Public Static FinalString Maxline_key = "Maxlinelength";  Public Static Final intMaxline_dflt = 2048; PrivateStringBuffer Eventstringbuffer =NewStringBuffer (); Mylinedeserializer (context context, resettableinputstream in) { This. in =In ;  This. Outputcharset =Charset.forname (context.getstring (Out_charset_key, Charset_dflt));  This. Maxlinelength =Context.getinteger (Maxline_key, Maxline_dflt);  This. IsOpen =true; }    /*** Reads a line from a file and returns an event * *@returnEvent containing parsed line *@throwsIOException*/@Override PublicEvent readevent ()throwsIOException {ensureopen (); String Line=ReadLine (); Event Event=NULL;  while(Line! =NULL) {            //start with one timestamp, event end            if(Line.trim (). StartsWith ("20") {Event=Eventbuilder.withbody (eventstringbuffer.tostring (), outputcharset); Eventstringbuffer.delete (0, Eventstringbuffer.length ()); }            //Add Current line push to buffer            if(Line.trim (). Length () > 0) {                if(Eventstringbuffer.length () > 0) {eventstringbuffer.append (System.lineseparator ()). append (line); } Else{eventstringbuffer.append (line); }            }            if(Line.trim (). StartsWith ("20")) {                 Break; } Line=ReadLine (); }        if(line = =NULL&& eventstringbuffer.tostring (). Length () > 0) {Event=Eventbuilder.withbody (eventstringbuffer.tostring (), outputcharset); Eventstringbuffer.delete (0, Eventstringbuffer.length ()); returnevent; }        returnevent; }    /*** Batch Line Read * *@paramnumevents Maximum Number of events to return. * @returnList of events containing read lines *@throwsIOException*/@Override PublicList<event> readevents (intnumevents)throwsIOException {ensureopen (); List<Event> events =lists.newlinkedlist ();  for(inti = 0; i < numevents; i++) {Event Event=readevent (); if(Event! =NULL) {Events.add (event); } Else {                 Break; }        }        returnevents; } @Override Public voidMark ()throwsIOException {ensureopen ();    In.mark (); } @Override Public voidReset ()throwsIOException {ensureopen ();    In.reset (); } @Override Public voidClose ()throwsIOException {if(isOpen) {reset ();            In.close (); IsOpen=false; }    }    Private voidEnsureopen () {if(!IsOpen) {            Throw NewIllegalStateException ("Serializer has been closed"); }    }    //Todo:consider not returning a final character that's a high surrogate//When truncating    PrivateString ReadLine ()throwsIOException {StringBuilder sb=NewStringBuilder (); intC; intReadChars = 0;  while((c = In.readchar ())! = 1) {ReadChars++; //fixme:support \ r \ n            if(c = = ' \ n ') {                 Break; } sb.append ((Char) c); if(ReadChars >=maxlinelength) {Logger.warn ("Line length exceeds Max ({}), truncating line!", maxlinelength);  Break; }        }        if(ReadChars > 0) {            returnsb.tostring (); } Else {            return NULL; }    }     Public Static classBuilderImplementsEventdeserializer.builder {@Override PublicMylinedeserializer Build (context context, resettableinputstream in) {return NewMylinedeserializer (context, in); }    }}

2:flume configuration file

A1.sources.r1.deserializer =  Com.xxx.flume.serializer.mylinedeserializer$builder

Flume Customizing the Deserializer Deserializer