Zookeeper Practice: Implementing a consumer process allocation program through zookeeper

background

Need to implement a distributed monitoring program, the agent sends the monitoring message to Kafka, and the consumer process uses the High level API to get the message from Kafka

Demand

As the number of agents increases, the number of messages generated per second is increasing, taking into account that the high-level API makes the consumer process a fixed relationship with the partition, so you need to use zookeeper to maintain a consumer's cluster, Enables consumers to dynamically allocate the number of consumer processes running on each consumer node based on the number of Kafka partitions and the number of consumer clusters

Implement

Package Com.eric.hadoop.zookeeper.coordination;import Java.net.inetaddress;import java.net.UnknownHostException; Import Java.util.hashmap;import java.util.list;import Java.util.map;import Org.apache.curator.retrypolicy;import Org.apache.curator.framework.curatorframework;import org.apache.curator.framework.CuratorFrameworkFactory; Import Org.apache.curator.framework.recipes.cache.nodecache;import Org.apache.curator.framework.recipes.cache.nodecachelistener;import Org.apache.curator.framework.recipes.cache.pathchildrencache;import Org.apache.curator.framework.recipes.cache.pathchildrencacheevent;import Org.apache.curator.framework.recipes.cache.pathchildrencachelistener;import Org.apache.curator.retry.exponentialbackoffretry;import Org.apache.curator.utils.ensurepath;import Org.apache.zookeeper.createmode;import org.codehaus.plexus.util.stringutils;/** * Depending on the number of Kafkaconsumermanager runs, Dynamic allocation runs the number of consumer * * @author Aihua.sun */public class Kafkaconsumermanager {private static final String Kafka_part_count = "Ten";p rivate static final String IPS = "localhost:2181";p rivate static final int session_timeout = 500 0;private static final int connect_timeout = 5000;private static final String NAMESPACE = "Monitor_managers";p rivate Stati c Final String kafka_path = "/kafka_count";p rivate static final String monitor_nodes_path = "/nodes";p rivate static final String alloc_node = "/alloc_node";p rivate string tempnodeid;private string oldconsumerthreadcount = "0";p rivate Curatorframework client;public static void Main (string[] args) {Kafkaconsumermanager Kafkaconsumermanager = new Kafkaconsumermanager (); Kafkaconsumermanager.initclient (); Kafkaconsumermanager.startconsumerthreads ();} public void Initclient () {Retrypolicy retrypolicy = new Exponentialbackoffretry (3); client = Curatorframeworkfactory.builder (). ConnectString (IPS). SESSIONTIMEOUTMS (Session_timeout). ConnectionTimeoutMs ( connect_timeout). Retrypolicy (Retrypolicy). Namespace (namespace). build (); Client.start ();} public void StarTconsumerthreads () {registerkafkapartcount (); Registermonitornodeintocluster (); Registerchildchangelistener (); Registerdatachangelistener (); try {thread.sleep (integer.max_value);} catch (Interruptedexception e) { E.printstacktrace ();}} private void Registerkafkapartcount () {Ensurepath Ensurepath = new Ensurepath ("/" + NAMESPACE + Kafka_path); try {Ensurepat H.ensure (Client.getzookeeperclient ()); Client.setdata (). Forpath (Kafka_path, Kafka_part_count.getbytes ());} catch (Exception e) {e.printstacktrace ();}} private void Registermonitornodeintocluster () {inetaddress Address;try {address = Inetaddress.getlocalhost (); Tempnodeid = "/" + address.gethostname () + System.currenttimemillis (); Ensurepath Ensurenodespath = new Ensurepath ("/" + NAMESPACE + monitor_nodes_path); Ensurepath Ensureallocpath = new Ensurepath ("/" + NAMESPACE + alloc_node); Ensurenodespath.ensure ( Client.getzookeeperclient ()); Ensureallocpath.ensure (Client.getzookeeperclient ());//When creating a node, the default is to put 0client.create () . Withmode (Createmode.ephemeral). Forpath (Monitor_nodes_path + Tempnodeid, "0". GetBytes ()); Client.setdata (). Forpath (Alloc_node, Tempnodeid.getbytes ()); SYSTEM.OUT.PRINTLN ("Send Add node Request:" + Tempnodeid);} catch (Unknownhostexception e) {e.printstacktrace ();} catch (Exception e) {e.printstacktrace ()}} /** * Node Change listener when the number of nodes changes, reassign the number of consumer processes running per node */private void Registerchildchangelistener () {try {final Pathchildrencache Nodecache = new Pathchildrencache (client, Monitor_nodes_path, true); Nodecache.start (); Nodecache.getlistenable (). AddListener (New Pathchildrencachelistener () {@Overridepublic void childevent ( Curatorframework client, Pathchildrencacheevent event) throws Exception {string allocnode = new String (Client.getdata (). Forpath (Alloc_node))///Only the newly added dot will be assigned the number of consumer threads of the node switch (Event.gettype ()) {Case CHILD_ADDED:System.out.println ("Add new node : "+ event.getdata (). GetPath ()); Realloctionconsumer (); Break;case CHILD_REMOVED:System.out.println (" Delete node: "+ Event.getdata (). GetPath ()); Realloctionconsumer (); break;default:break;}});} CatcH (Exception e) {e.printstacktrace ();}} Reassign the number of consumer processes initiated per Monitor-manager private void Realloctionconsumer () {String Kafkapartcountstr;try {kafkapartcountstr = New String (Client.getdata (). Forpath (Kafka_path)); list<string> childrenlist = Client.getchildren (). Forpath (monitor_nodes_path); int childrenlistsize = Childrenlist.size (); if (Stringutils.isalphanumeric (kafkapartcountstr) && childrenlistsize > 0) {int Kafkapartcount = integer.valueof (KAFKAPARTCOUNTSTR); map<string, integer> nodeconsuemercounter = new hashmap<string, integer> (); for (int i = 1; I <= kafkaPartCo Unt i++) {String nodeName = childrenlist.get (i% childrenlistsize); if (Nodeconsuemercounter.get (nodeName) = = null) { Nodeconsuemercounter.put (NodeName, 1);} else {int oldpartcount = Nodeconsuemercounter.get (nodeName); Nodeconsuemercounter.put (NodeName, Oldpartcount + 1);}} System.out.println ("Number of consumer processes run by each host after Reallocation"); for (String NodeName:nodeConsuemerCounter.keySet ()) {Integer Consumerthreadcount = NodeCOnsuemercounter.get (NodeName); System.out.println ("Node name:" + nodeName + "Number of consumer processes:" + consumerthreadcount); Client.setdata (). Forpath (Monitor_nodes_path + "/" + nodename,consumerthreadcount.tostring (). GetBytes ());}} catch (Exception e) {e.printstacktrace ();}} private void Registerdatachangelistener () {try {final nodecache Nodecache = new Nodecache (client, Monitor_nodes_path + tem Pnodeid, False); Nodecache.start (); nodecache.getlistenable (). AddListener (New Nodecachelistener () {@Overridepublic void NodeChanged () throws Exception {string newconsumerthreadcountstr = new String (Nodecache.getcurrentdata (). GetData ( ) or//the corresponding operation if (!oldconsumerthreadcount.equals (NEWCONSUMERTHREADCOUNTSTR)) {if () only if the new number of nodes is changed Stringutils.isnumeric (NEWCONSUMERTHREADCOUNTSTR)) {System.out.println ("the number of new consumer processes on this host is:" + NEWCONSUMERTHREADCOUNTSTR); oldconsumerthreadcount = Newconsumerthreadcountstr;} else {System.out.println ("the number of illegal consumer processes on this host has been generated:" + Newconsumerthreadcountstr);}}});} catch (Exception e) {e.printstacktrace ();}}}

Zookeeper Practice: Implementing a consumer process allocation program through zookeeper