Flume Learning 07-flumerpcclientutils Tool class

The flumerpcclientutils provides a common event operation feature that allows you to switch between multiple rpcclient through a configuration file.

Flumerpcclientutils Configuration Parameters

# optional value default, Thrift, Default_failover, Default_loadbalance # where default uses the AVRO protocol # if default is used _failover and Default_loadbalance, then can only use Avro protocol Client.type = default # If the Client.type value is default or thrift, then hosts only have one host #
If the Client.type value is Default_failover or default_loadbalance, then the hosts must configure at least two hosts. hosts = h1 # hosts = h1,h2,h3 # host Configuration hosts.h1 = 127.0.0.1:41414 # hosts.h2 = host2.example.org:41414 # hosts.h3 = host3.e
xample.org:41414 # If the host connection fails, the host is temporarily blacklisted, and then enabled for a period of time.
# defaults to False, not blacklisted.
Backoff = False # Host connection failed, the maximum time interval for the next connection to the host, in milliseconds.
# The default value is 0, which is equivalent to 30000 milliseconds.
Maxbackoff = 0 # If Client.type uses Default_loadbalance, the host's polling policy is in this mode.
# Optional policies have Round_robin, random, or custom policies (implementing the Loadbalancingrpcclient$hostselector interface).
# The default is Round_robin host-selector = round_robin # Number of batches sent, the value must be greater than 0, the default is 100.
batch-size = 500 # Connection Timeout, the value must be greater than 1000, default is 20000, in milliseconds.
Connect-timeout = 20000 # The request timed out, the value must be greater than 1000, the default is 20000, in milliseconds. Request-timeout = 20000 # message encoding, default is Utf-8 charset = utf-8 # If the message failed to send, the number of messages attempted to send, the default is 3 times Attempttimes = 3 

Import java.io.IOException;
Import Java.io.InputStream;
Import Java.nio.charset.Charset;
Import java.util.ArrayList;
Import java.util.LinkedList;
Import java.util.List;
Import Java.util.Map;

Import java.util.Properties;
Import org.apache.flume.Event;
Import org.apache.flume.EventDeliveryException;
Import org.apache.flume.api.RpcClient;
Import Org.apache.flume.api.RpcClientFactory;
Import Org.apache.flume.event.EventBuilder;
Import Org.slf4j.Logger;

Import Org.slf4j.LoggerFactory; /** * Flume Client Tool class * Tool class Initialize default read profile flume-client.properties * config file placed under classpath * * @author accountwcx@qq.com * */PU Blic class Flumerpcclientutils {private static final Logger Logger = Loggerfactory.getlogger (flumerpcclientutils.clas

    s);

    private static properties props = new properties ();

    Whether to initialize private static volatile for Boolean isinit = false;

    The default encoding for sending messages, and if no encoding is set, use the default encoding private static final String Defaultcharsetname = "Utf-8"; Encoding of the sending message private static CharsetCharSet

    If the message fails to send, the number of messages attempted to be sent, the default is 3 times private static int attempttimes = 3;

    private static rpcclient client; 
        /** * Initialize Client configuration Client configuration must be placed in the Flume-client.properties file in the classpath root directory */public synchronized static void Init () {
        if (isinit) {return;

        } logger.info ("Initialize configuration Flume-client.properties"); Read configuration file InputStream is = FlumeRpcClientUtils.class.getClassLoader (). getResourceAsStream ("Flume-client.properties

        ");
            if (is = = null) {logger.error ("configuration file flume-client.properties not found");
        Return

            } try {props.load (IS);
            Reads the message encoding from the configuration file String CharsetName = Props.getproperty ("charset"); if (charsetname! = null &&!charsetname.equals ("")) {try {charset = CharSet.
                Forname (CharsetName); } catch (Exception e) {logger.error ("encoding charset=" + CharsetName + "initialInitialization failed with default encoding charset= "+ Defaultcharsetname, E);

            }} props.remove ("CharSet"); If the encoding is empty, use the default encoding utf-8 if (charset = = null) {try {charset = Charset.forn
                Ame (Defaultcharsetname);
                } catch (Exception e) {logger.error ("default encoding charset=" + Defaultcharsetname + "initialization failed", e);
            }}//Read the number of messages sent configuration String strattempttimes = Props.getproperty ("Attempttimes");
                if (strattempttimes! = null &&!strattempttimes.equals ("")) {int tmpattempttimes = 0;
                try {tmpattempttimes = Integer.parseint (strattempttimes); } catch (NumberFormatException e) {logger.error ("message sent attempttimes=" + Strattempttimes + "initialization failed with default
                Sent attempttimes= "+ attempttimes, E);
         } if (Tmpattempttimes > 0) {           Attempttimes = Tmpattempttimes;

            }} props.remove ("Attempttimes");

            Initialize flume client//According to different Client.type, the instance is not the same as Client = Rpcclientfactory.getinstance (props);
        Isinit = true;
        } catch (IOException e) {logger.error ("config file flume-client.properties read failed", e);
                } finally {if (is = null) {try {is.close (); } catch (IOException e) {}}}/** * Sends a record, if the send fails, the method attempts to send multiple times, the number of attempts at
     Set in Tempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance.
        * * @param data Send content * @return Send successful return true, Failure returns false */public static Boolean append (String data) {
        Boolean flag = false;
        Event event = eventbuilder.withbody (data, CharSet);

        int current = 0;
          while (!flag && current < attempttimes) {current++;  try {client.append (event);
            Flag = true;
                } catch (Eventdeliveryexception e) {logger.error ("Send failed, currently tried" + current + "times", e);
            Logger.error ("failed message" + data);
    }} return flag;
     /** * Send a record, if send failed, the method will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance.
        * * @param event Send content * @return Send successful return true, Failure returns false */public static Boolean append (Event event) {

        Boolean flag = false;

        int current = 0;
            while (!flag && current < attempttimes) {current++;
                try {client.append (event);
            Flag = true;
                } catch (Eventdeliveryexception e) {logger.error ("Send failed, currently tried" + current + "times", e);
            Logger.error ("failed message" + New String (Event.getbody (), CharSet));
    }} return flag; }/** * Sends a record, ifSend failed, the method will attempt to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance. * * @param data Send content * @param headers sent header file * @return Send successful return true, Failure returns false */public static bool

        EAN append (String data, map<string, string> headers) {Event event = eventbuilder.withbody (data, CharSet);
        if (headers! = null) {event.setheaders (headers);
    } return append (event);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * * Appendbatch performance is much higher than append, recommended. 
        * * @param data single record * @return send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (String data) {
        list<string> items = new arraylist<string> ();
        Items.Add (data);
    return Appendbatch (items);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times. 
*     * Appendbatch performance is much higher than append, it is recommended. * * @param data single record * @return send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (String data,
        Map<string, string> headers) {list<event> events = new arraylist<event> ();
        Event event = eventbuilder.withbody (data, CharSet);
        Event.setheaders (headers);
        Events.add (event);
    Return Appendbatchevent (events);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * * Appendbatch performance is much higher than append, recommended. 
        * * @param data single record * @return send successful return true, Failure returns false */public static Boolean Appendbatch (Event event) {
        list<event> events = new arraylist<event> ();
        Events.add (event);
    Return Appendbatchevent (events);
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param itemsTable of Contents * @return Send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (list<string> items) {
    Return Appendbatch (items, null);
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param items Content list * @param headers header content * @return send successfully returns TRUE, Failure returns false */public static Boole

        An Appendbatch (list<string> items, map<string, string> headers) {Boolean flag = false;
        If the parameter does not meet the requirements, exit if (items = null | | items.size () < 1) {return flag;

        } list<event> Events = new linkedlist<event> (); if (headers! = null) {for (String Item:items) {Event event = Eventbuilder.withbody (item, chars
                ET);
                Event.setheaders (headers);
            Events.add (event); }}else{for (String item:items) {Events.add (Eventbuilder.witHbody (item, CharSet));

        }}//current attempt to send the number of times int = 0;
            while (!flag && current < attempttimes) {current++;
                try {Client.appendbatch (events);
            Flag = true;
            } catch (Eventdeliveryexception e) {logger.error ("Bulk send failed, currently tried" + current + "times", e);
    }} return flag;
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param events Content list * @return Send successfully returns TRUE, Failure returns false */public static Boolean appendbatchevent (LIST&L T

        Event> events) {Boolean flag = false;
        If the parameter does not meet the requirements, exit if (events = null | | events.size () < 1) {return flag;

        }//Current attempt to send the number of times int = 0;
            while (!flag && current < attempttimes) {current++; try {ClieNt.appendbatch (events);
            Flag = true;
            } catch (Eventdeliveryexception e) {logger.error ("Bulk send failed, currently tried" + current + "times", e);
    }} return flag;
    } public static int getbatchsize () {return client.getbatchsize ();
 }
}

Sample code

Initialize
flumerpcclientutils.init ();

for (int i = 0; i < i++) {String Data
    = "Send" + i + "strip";
    Flumerpcclientutils.append (data);
}