Storm---directgroup (direct grouping)

Take the word segmentation count as an example to implement Storm's Directgroup grouping:

1, spout realization

Spout is the source of storm data, using the Directgroup method to send spout data to specified bolts, note:

1), spout consumes the Bolt's task (task should be the number of the storm's executor), in the following code Spout.open () initialization to get the consuming task

2), need to use the Spoutoutputcollector.emitdirect () method

3), declare the spout as a direct stream, that is, in the Spout.declareoutputfields () declaration

/** * Fixed Cycle Spout * * @author Hanhan.zhang * */public class Fixedcyclespout implements Irichspout {private Strin    G _fieldname;    Private Boolean _direct;    Stream Mark private String _streamid;    private int _index;    Key = MsgId, value = sending tuple private map<string, list<object>> _pendingtuple;    Send tuple private list<object> [] _sendtuple;    Private Spoutoutputcollector _collector;    Private Countmetric _sendmetric;    Private Countmetric _failmetric;    Consume task set private list<integer> _consumetaskidlist; Public Fixedcyclespout (String _streamid, String _fieldname, Boolean _direct, list<object> ... _sendtuple) {T        His._streamid = _streamid;        This._fieldname = _fieldname;        This._direct = _direct;    This._sendtuple = _sendtuple;  } @Override public void Open (Map conf, topologycontext context, Spoutoutputcollector collector) {This._index        = 0; _pendingtuple= Maps.newhashmap (); Register metric this._sendmetric = Context.registermetric ("Cycle.spout.send.tuple.metric", New Countmetric (), 60        );        This._failmetric = Context.registermetric ("Cycle.spout.fail.tuple.metric", New Countmetric (), 60);        This._collector = collector;            Get consume task ID if (this._direct) {this._consumetaskidlist = Lists.newlinkedlist ();            Map<string, map<string, grouping>> consumetargets = Context.getthistargets ();                if (consumetargets! = null &&!consumetargets.isempty ()) {//Streamid = This._streamid Consumetargets.foreach (Streamid, target), {if (target! = null &&!target.isempty ()) {//ComponentID = Consume target component Id Target.foreach (ComponentID, Group), {if (Group.is_set_direct ()) {This._coNsumetaskidlist.addall (Context.getcomponenttasks (ComponentID));                    }                        });            }                }); }}} @Override public void Close () {} @Override public void Activate () {} @Override P        ublic void Deactivate () {} @Override public void Nexttuple () {this._sendmetric.incr ();        if (This._index = = _sendtuple.length) {this._index = 0;        } String msgId = Uuid.randomuuid (). toString ();        List<object> tuple = this._sendtuple[this._index++];    Sendtuple (msgId, tuple);        } @Override public void ack (Object msgId) {String msgidstr = (string) msgId;        SYSTEM.OUT.PRINTLN ("Ack tuple with msgId" + msgidstr);    This._pendingtuple.remove (MSGIDSTR);        } @Override public void fail (Object msgId) {this._failmetric.incr ();        String msgidstr = (string) msgId; SYSTEM.OUT.PRINTLN ("Fail tuple WITh msgId "+ msgidstr);    Sendtuple (Msgidstr, This._pendingtuple.get (MSGIDSTR)); } @Override public void Declareoutputfields (Outputfieldsdeclarer declarer) {Declarer.declarestream (this._str    Eamid, This._direct, new Fields (_fieldname));    } @Override public map<string, object> getcomponentconfiguration () {return null;        } protected void Sendtuple (String msgId, list<object> tuple) {this._pendingtuple.put (msgId, tuple);                if (this._direct) {if (this._consumetaskidlist = = NULL | | this._consumetaskidlist.isempty ()) {            throw new IllegalStateException ("Direct task is empty!"); } this._consumetaskidlist.foreach (TaskId, This._collector.emitdirect (TaskId, This._stre        AmId, tuple, msgId));        } else {this._collector.emit (tuple, msgId); }    }}

2. Bolt implementation

/*** Sentence Split Bolt * *@authorHanhan.zhang **/ Public classSentencesplitboltImplementsIrichbolt {PrivateOutputcollector _collector; PrivateCountmetric _ackmetric; PrivateCountmetric _failmetric; PrivateString _separator; Private int_taskid; Private Boolean_direct; PrivateString _streamid;  PublicSentencesplitbolt (String _streamid,Boolean_direct) {         This. _streamid =_streamid;  This. _direct =_direct; }    /**     * @paramcontext * 1:register Metric * 2:next Bolt Message *@paramCollector (thread-safe) * 1:emit tuple * 2:ack/fail tuple **/     Public voidPrepare (Map stormconf, Topologycontext context, Outputcollector collector) { This. _collector =collector; //register metric for Monitor         This. _ackmetric = Context.registermetric ("Sentence.split.ack.metric",NewCountmetric (), 60);  This. _failmetric = Context.registermetric ("Sentence.split.fail.metric",NewCountmetric (), 60);  This. _taskid =Context.getthistaskid ();  This. _separator =(String) stormconf.get (const.separator); } @Override Public voidExecute (Tuple input) {Try{String sentence= input.getstring (0); if(strings.isnullorempty (sentence)) {return; } String []fields=Sentence.split (_separator);  for(String field:fields) {if( This. _direct) {                     This. _collector.emitdirect ( This. _taskid, _streamid, input,NewValues (field, 1)); } Else {                     This. _collector.emit ( This. _streamid, input,NewValues (field, 1)); }            }             This. _collector.ack (input);  This. _ACKMETRIC.INCR (); } Catch(Exception e) { This. _collector.fail (input);  This. _FAILMETRIC.INCR (); }} @Override Public voidCleanup () {} @Override Public voiddeclareoutputfields (Outputfieldsdeclarer declarer) {Declarer.declarestream ( This. _streamid, This. _direct,NewFields ("word", "count")); } @Override PublicMap<string, object>getcomponentconfiguration () {return NULL; }}/*** Word Sum Bolt * *@authorHanhan.zhang **/ Public classWordsumboltextendsBaserichbolt {PrivateOutputcollector _collector; Private int_taskid; PrivateCache<string, atomicinteger>_wordcache; @Override Public voidPrepare (Map stormconf, Topologycontext context, Outputcollector collector) { This. _collector =collector;  This. _taskid =Context.getthistaskid ();  This. _wordcache =Cachebuilder.newbuilder (). MaximumSize (1024). Expireafterwrite (3, Timeunit.seconds). Removallistener (removalnotification)-{String key=(String) Removalnotification.getkey (); Atomicinteger sum=(Atomicinteger) removalnotification.getvalue (); System.out.println ("Word sum result: [" + Key + "," + sum.get () + "]");    }). build (); } @Override Public voidExecute (Tuple input) {Try{String Word= input.getstring (0); intCount = Input.getinteger (1); if(Strings.isempty (Word)) {return; } Atomicinteger Counter= This. _wordcache.getifpresent (word); if(Counter = =NULL) {                 This. _wordcache.put (Word,NewAtomicinteger (count)); } Else{counter.addandget (count); }             This. _collector.ack (input); } Catch(Exception e) { This. _collector.fail (input); }} @Override Public voiddeclareoutputfields (Outputfieldsdeclarer declarer) {}}

3. Storm Run

/*** Tuple split-flow topology * *@authorHanhan.zhang **/ Public classFlowtopology { Public Static voidMain (string[] args) {//Send tuplelist<object> []tuple =NewList[] {NewValues ("The cow jumped over the moon"),                                            NewValues ("The man went to the store and bought some candy"),                                            NewValues ("Four score and seven years ago"),                                            NewValues ("How many apples can eat")}; //Stream nameString spoutstreamid = "Topology.flow.cycle.spout.stream"; String Splitstreamid= "Topology.flow.split.bolt.stream"; //SpoutFixedcyclespout cyclespout =NewFixedcyclespout (Spoutstreamid, "sentence",true, tuple); //BoltSentencesplitbolt Splitbolt =NewSentencesplitbolt (Splitstreamid,false); Wordsumbolt Sumbolt=NewWordsumbolt (); Topologybuilder Topologybuilder=NewTopologybuilder (); Topologybuilder.setspout ("Sentence.cycle.spout", Cyclespout, 1); Topologybuilder.setbolt ("Sentence.split.bolt", Splitbolt, 1). directgrouping ("Sentence.cycle.spout", Spoutstreamid); Topologybuilder.setbolt ("Word.sum.bolt", Sumbolt, 3). fieldsgrouping ("Sentence.split.bolt", Splitstreamid,NewFields ("word")); Config config=NewConfig (); Config.put (Const.separator," "); Localcluster Localcluster=NewLocalcluster (); Localcluster.submittopology ("Flowtopology", config, topologybuilder.createtopology ()); }}

Storm---directgroup (direct grouping)