Flume Learning 07-flumerpcclientutils Tool class

Source: Internet
Author: User
Tags config time interval root directory

The flumerpcclientutils provides a common event operation feature that allows you to switch between multiple rpcclient through a configuration file.

Flumerpcclientutils Configuration Parameters

# optional value default, Thrift, Default_failover, Default_loadbalance # where default uses the AVRO protocol # if default is used _failover and Default_loadbalance, then can only use Avro protocol Client.type = default # If the Client.type value is default or thrift, then hosts only have one host #
If the Client.type value is Default_failover or default_loadbalance, then the hosts must configure at least two hosts. hosts = h1 # hosts = h1,h2,h3 # host Configuration hosts.h1 = 127.0.0.1:41414 # hosts.h2 = host2.example.org:41414 # hosts.h3 = host3.e
xample.org:41414 # If the host connection fails, the host is temporarily blacklisted, and then enabled for a period of time.
# defaults to False, not blacklisted.
Backoff = False # Host connection failed, the maximum time interval for the next connection to the host, in milliseconds.
# The default value is 0, which is equivalent to 30000 milliseconds.
Maxbackoff = 0 # If Client.type uses Default_loadbalance, the host's polling policy is in this mode.
# Optional policies have Round_robin, random, or custom policies (implementing the Loadbalancingrpcclient$hostselector interface).
# The default is Round_robin host-selector = round_robin # Number of batches sent, the value must be greater than 0, the default is 100.
batch-size = 500 # Connection Timeout, the value must be greater than 1000, default is 20000, in milliseconds.
Connect-timeout = 20000 # The request timed out, the value must be greater than 1000, the default is 20000, in milliseconds. Request-timeout = 20000 # message encoding, default is Utf-8 charset = utf-8 # If the message failed to send, the number of messages attempted to send, the default is 3 times Attempttimes = 3 


Import java.io.IOException;
Import Java.io.InputStream;
Import Java.nio.charset.Charset;
Import java.util.ArrayList;
Import java.util.LinkedList;
Import java.util.List;
Import Java.util.Map;

Import java.util.Properties;
Import org.apache.flume.Event;
Import org.apache.flume.EventDeliveryException;
Import org.apache.flume.api.RpcClient;
Import Org.apache.flume.api.RpcClientFactory;
Import Org.apache.flume.event.EventBuilder;
Import Org.slf4j.Logger;

Import Org.slf4j.LoggerFactory; /** * Flume Client Tool class * Tool class Initialize default read profile flume-client.properties * config file placed under classpath * * @author accountwcx@qq.com * */PU Blic class Flumerpcclientutils {private static final Logger Logger = Loggerfactory.getlogger (flumerpcclientutils.clas

    s);

    private static properties props = new properties ();

    Whether to initialize private static volatile for Boolean isinit = false;

    The default encoding for sending messages, and if no encoding is set, use the default encoding private static final String Defaultcharsetname = "Utf-8"; Encoding of the sending message private static CharsetCharSet

    If the message fails to send, the number of messages attempted to be sent, the default is 3 times private static int attempttimes = 3;

    private static rpcclient client; 
        /** * Initialize Client configuration Client configuration must be placed in the Flume-client.properties file in the classpath root directory */public synchronized static void Init () {
        if (isinit) {return;

        } logger.info ("Initialize configuration Flume-client.properties"); Read configuration file InputStream is = FlumeRpcClientUtils.class.getClassLoader (). getResourceAsStream ("Flume-client.properties

        ");
            if (is = = null) {logger.error ("configuration file flume-client.properties not found");
        Return

            } try {props.load (IS);
            Reads the message encoding from the configuration file String CharsetName = Props.getproperty ("charset"); if (charsetname! = null &&!charsetname.equals ("")) {try {charset = CharSet.
                Forname (CharsetName); } catch (Exception e) {logger.error ("encoding charset=" + CharsetName + "initialInitialization failed with default encoding charset= "+ Defaultcharsetname, E);

            }} props.remove ("CharSet"); If the encoding is empty, use the default encoding utf-8 if (charset = = null) {try {charset = Charset.forn
                Ame (Defaultcharsetname);
                } catch (Exception e) {logger.error ("default encoding charset=" + Defaultcharsetname + "initialization failed", e);
            }}//Read the number of messages sent configuration String strattempttimes = Props.getproperty ("Attempttimes");
                if (strattempttimes! = null &&!strattempttimes.equals ("")) {int tmpattempttimes = 0;
                try {tmpattempttimes = Integer.parseint (strattempttimes); } catch (NumberFormatException e) {logger.error ("message sent attempttimes=" + Strattempttimes + "initialization failed with default
                Sent attempttimes= "+ attempttimes, E);
         } if (Tmpattempttimes > 0) {           Attempttimes = Tmpattempttimes;

            }} props.remove ("Attempttimes");

            Initialize flume client//According to different Client.type, the instance is not the same as Client = Rpcclientfactory.getinstance (props);
        Isinit = true;
        } catch (IOException e) {logger.error ("config file flume-client.properties read failed", e);
                } finally {if (is = null) {try {is.close (); } catch (IOException e) {}}}/** * Sends a record, if the send fails, the method attempts to send multiple times, the number of attempts at
     Set in Tempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance.
        * * @param data Send content * @return Send successful return true, Failure returns false */public static Boolean append (String data) {
        Boolean flag = false;
        Event event = eventbuilder.withbody (data, CharSet);

        int current = 0;
          while (!flag && current < attempttimes) {current++;  try {client.append (event);
            Flag = true;
                } catch (Eventdeliveryexception e) {logger.error ("Send failed, currently tried" + current + "times", e);
            Logger.error ("failed message" + data);
    }} return flag;
     /** * Send a record, if send failed, the method will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance.
        * * @param event Send content * @return Send successful return true, Failure returns false */public static Boolean append (Event event) {

        Boolean flag = false;

        int current = 0;
            while (!flag && current < attempttimes) {current++;
                try {client.append (event);
            Flag = true;
                } catch (Eventdeliveryexception e) {logger.error ("Send failed, currently tried" + current + "times", e);
            Logger.error ("failed message" + New String (Event.getbody (), CharSet));
    }} return flag; }/** * Sends a record, ifSend failed, the method will attempt to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * It is recommended to use Appendbatch for better performance. * * @param data Send content * @param headers sent header file * @return Send successful return true, Failure returns false */public static bool

        EAN append (String data, map<string, string> headers) {Event event = eventbuilder.withbody (data, CharSet);
        if (headers! = null) {event.setheaders (headers);
    } return append (event);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * * Appendbatch performance is much higher than append, recommended. 
        * * @param data single record * @return send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (String data) {
        list<string> items = new arraylist<string> ();
        Items.Add (data);
    return Appendbatch (items);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times. 
*     * Appendbatch performance is much higher than append, it is recommended. * * @param data single record * @return send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (String data,
        Map<string, string> headers) {list<event> events = new arraylist<event> ();
        Event event = eventbuilder.withbody (data, CharSet);
        Event.setheaders (headers);
        Events.add (event);
    Return Appendbatchevent (events);
     /** * Sends a record in bulk, which is not sent immediately, but is placed in the memory queue until the number of records in the queue reaches batchsize.
     * If send failed, will try to send multiple times, the number of attempts set in Attempttimes, default 3 times.
     * * Appendbatch performance is much higher than append, recommended. 
        * * @param data single record * @return send successful return true, Failure returns false */public static Boolean Appendbatch (Event event) {
        list<event> events = new arraylist<event> ();
        Events.add (event);
    Return Appendbatchevent (events);
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param itemsTable of Contents * @return Send successfully returns TRUE, Failure returns false */public static Boolean Appendbatch (list<string> items) {
    Return Appendbatch (items, null);
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param items Content list * @param headers header content * @return send successfully returns TRUE, Failure returns false */public static Boole

        An Appendbatch (list<string> items, map<string, string> headers) {Boolean flag = false;
        If the parameter does not meet the requirements, exit if (items = null | | items.size () < 1) {return flag;

        } list<event> Events = new linkedlist<event> (); if (headers! = null) {for (String Item:items) {Event event = Eventbuilder.withbody (item, chars
                ET);
                Event.setheaders (headers);
            Events.add (event); }}else{for (String item:items) {Events.add (Eventbuilder.witHbody (item, CharSet));

        }}//current attempt to send the number of times int = 0;
            while (!flag && current < attempttimes) {current++;
                try {Client.appendbatch (events);
            Flag = true;
            } catch (Eventdeliveryexception e) {logger.error ("Bulk send failed, currently tried" + current + "times", e);
    }} return flag;
     /** * Bulk send multiple records, if send failed, will try to send multiple times, the number of attempts set in Attempttimes, the default 3 times.
     * Appendbatch performance is much higher than append, it is recommended. * * @param events Content list * @return Send successfully returns TRUE, Failure returns false */public static Boolean appendbatchevent (LIST&L T

        Event> events) {Boolean flag = false;
        If the parameter does not meet the requirements, exit if (events = null | | events.size () < 1) {return flag;

        }//Current attempt to send the number of times int = 0;
            while (!flag && current < attempttimes) {current++; try {ClieNt.appendbatch (events);
            Flag = true;
            } catch (Eventdeliveryexception e) {logger.error ("Bulk send failed, currently tried" + current + "times", e);
    }} return flag;
    } public static int getbatchsize () {return client.getbatchsize ();
 }
}


Sample code

Initialize
flumerpcclientutils.init ();

for (int i = 0; i < i++) {String Data
    = "Send" + i + "strip";
    Flumerpcclientutils.append (data);
}

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.