Import Java.io.File;
Import java.io.IOException;
Import java.util.Collection;
Import Java.util.HashMap;
Import java.util.List;
Import Java.util.Map;
Import Java.util.Map.Entry;
Import Org.apache.commons.io.FileUtils;
Import Backtype.storm.Config;
Import Backtype.storm.LocalCluster;
Import Backtype.storm.spout.SpoutOutputCollector;
Import Backtype.storm.task.OutputCollector;
Import Backtype.storm.task.TopologyContext;
Import Backtype.storm.topology.OutputFieldsDeclarer;
Import Backtype.storm.topology.TopologyBuilder;
Import Backtype.storm.topology.base.BaseRichBolt;
Import Backtype.storm.topology.base.BaseRichSpout;
Import Backtype.storm.tuple.Fields;
Import Backtype.storm.tuple.Tuple;
Import backtype.storm.tuple.Values;
Import Cn.crxy.storm.LocalStormTopology.SumBolt;
public class Wordcountstormtopology {
public static class Datasourcespout extends baserichspout{
Private MAP conf;
Private Topologycontext context;
Private Spoutoutputcollector collector;
/**
* Called once when the instance is running
*/
public void Open (Map conf, Topologycontext context,
Spoutoutputcollector collector) {
this.conf = conf;
This.context = context;
This.collector = collector;
}
/**
* Dead Loop call Heartbeat
*/
public void Nexttuple () {
Get all files under the specified folder
collection<file> files = fileutils.listfiles (new File ("D:\\test"), new string[]{"TXT"}, True);
for (File file:files) {
try {
Parse each line of a file
list<string> readlines = fileutils.readlines (file);
for (String line:readlines) {
Send out every row of data
This.collector.emit, new Values (line);
}
Renaming prevents multiple reads
Fileutils.movefile (file, new file (File.getabsolutepath () +system.currenttimemillis ()));
} catch (IOException e) {
E.printstacktrace ();
}
}
}
/**
* Declaring field names
*/
public void Declareoutputfields (Outputfieldsdeclarer declarer) {
field is the list of fields
Declarer.declare ("line");
}
}
public static class Spiltbolt extends baserichbolt{
Private Map stormconf;
Private Topologycontext context;
Private Outputcollector collector;
/**
* will only be called once
*/
public void Prepare (Map stormconf, Topologycontext context,
Outputcollector collector) {
this.stormconf = stormconf;
This.context = context;
This.collector = collector;
}
/**
* Dead loop, loop gets the data sent over the previous level (Spout/bolt)
*/
public void execute (Tuple input) {
Get the tuple sent data
String line = Input.getstringbyfield ("line");
Cut each row of data
string[] Words = Line.split ("\ t");
for (String word:words) {
Send the cut word to the next bolt
This.collector.emit (New Values (word));
}
}
public void Declareoutputfields (Outputfieldsdeclarer declarer) {
Declarer.declare (New fields ("word"));
}
}
public static class Countbolt extends baserichbolt{
Private Map stormconf;
Private Topologycontext context;
Private Outputcollector collector;
/**
* will only be called once
*/
public void Prepare (Map stormconf, Topologycontext context,
Outputcollector collector) {
this.stormconf = stormconf;
This.context = context;
This.collector = collector;
}
/**
* Dead loop, loop gets the data sent over the previous level (Spout/bolt)
*/
map<string, integer> map = new hashmap<string, integer> ();
public void execute (Tuple input) {
Get data sent in a tuple
String Word = Input.getstringbyfield ("word");
Integer value = Map.get (word);
if (value==null) {
value=0;
}
value++;
Save data to a Map object
Map.put (Word, value);
Write the results out.
System.err.println ("===============================");
For (entry<string, integer> entry:map.entrySet ()) {
SYSTEM.ERR.PRINTLN (entry);
}
}
public void Declareoutputfields (Outputfieldsdeclarer declarer) {
}
}
public static void Main (string[] args) {
Topologybuilder builder = new Topologybuilder ();
Builder.setspout ("1", New Datasourcespout ());
Builder.setbolt ("2", New Spiltbolt ()). Shufflegrouping ("1");
Builder.setbolt ("3", New Countbolt ()). Shufflegrouping ("2");
Localcluster localcluster = new Localcluster ();
Config config = new config ();
Localcluster.submittopology (WordcountStormTopology.class.getSimpleName (), config, builder.createtopology ());
}
}
Copyright NOTICE: This article for Bo Master original article, without Bo Master permission not reproduced.
Storm word Count Local execution
