Twitter Storm: DRPC學習

學習途徑作者: xumingming | 可以轉載, 但必須以超連結形式標明文章原始出處和作者資訊及著作權聲明
網址: http://xumingming.sinaapp.com/756/twitter-storm-drpc/

 

本文翻譯自：https://github.com/nathanmarz/storm/wiki/Distributed-RPC。

Storm裡面引入DRPC主要是利用storm的Realtime Compute能力來並行化CPU intensive的計算。DRPC的storm topology以函數的參數流作為輸入，而把這些函數調用的傳回值作為topology的輸出資料流。

DRPC其實不能算是storm本身的一個特性， 它是通過組合storm的原語spout，bolt， topology而成的一種模式(pattern)。本來應該把DRPC單獨打成一個包的， 但是DRPC實在是太有用了，所以我們我們把它和storm捆綁在一起。

DRPC例子建立一個Bolat作為資料處理

/** * ExclaimBolt.java * 著作權(C) 2013  * 建立:cuiran 2013-01-15 17:26:42 */package com.stormdemo.drpc;import java.util.Map;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import com.stormdemo.demo.DemoTopology;import junit.framework.TestCase;import backtype.storm.drpc.LinearDRPCTopologyBuilder;import backtype.storm.task.TopologyContext;import backtype.storm.topology.BasicOutputCollector;import backtype.storm.topology.IBasicBolt;import backtype.storm.topology.OutputFieldsDeclarer;import backtype.storm.tuple.Fields;import backtype.storm.tuple.Tuple;import backtype.storm.tuple.Values;/** * TODO * @author cuiran * @version TODO */public class ExclaimBolt   implements IBasicBolt {private static Log log = LogFactory.getLog(ExclaimBolt.class.getName());/* (non-Javadoc) * @see backtype.storm.topology.IBasicBolt#cleanup() */@Overridepublic void cleanup() {// TODO Auto-generated method stub}/* (non-Javadoc) * @see backtype.storm.topology.IBasicBolt#execute(backtype.storm.tuple.Tuple, backtype.storm.topology.BasicOutputCollector) */@Overridepublic void execute(Tuple tuple, BasicOutputCollector collector) {// TODO Auto-generated method stublog.debug("處理資料");String input = tuple.getString(1);      log.debug("接收到的資料為:"+input);collector.emit(new Values(tuple.getValue(0), input + "!")); }/* (non-Javadoc) * @see backtype.storm.topology.IBasicBolt#prepare(java.util.Map, backtype.storm.task.TopologyContext) */@Overridepublic void prepare(Map stormConf, TopologyContext context) {// TODO Auto-generated method stub}/* (non-Javadoc) * @see backtype.storm.topology.IComponent#declareOutputFields(backtype.storm.topology.OutputFieldsDeclarer) */@Overridepublic void declareOutputFields(OutputFieldsDeclarer declarer) {// TODO Auto-generated method stub declarer.declare(new Fields("id", "result")); }/* (non-Javadoc) * @see backtype.storm.topology.IComponent#getComponentConfiguration() */@Overridepublic Map<String, Object> getComponentConfiguration() {// TODO Auto-generated method stubreturn null;}}

然後寫一個測試類別

/** * DRPCTest.java * 著作權(C) 2013  * 建立:cuiran 2013-01-15 17:25:37 */package com.stormdemo.drpc;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import backtype.storm.Config;import backtype.storm.LocalCluster;import backtype.storm.LocalDRPC;import backtype.storm.drpc.LinearDRPCTopologyBuilder;import backtype.storm.topology.TopologyBuilder;import junit.framework.TestCase;/** * TODO * @author cuiran * @version TODO */public class DRPCTest extends TestCase {private static Log log = LogFactory.getLog(DRPCTest.class.getName());public void testDrpc(){log.debug("testDrpc開始");Config conf = new Config();conf.setDebug(true);conf.setNumWorkers(2);conf.setMaxSpoutPending(1);LocalDRPC drpc=new LocalDRPC();LocalCluster cluster=new LocalCluster();LinearDRPCTopologyBuilder builder  = new LinearDRPCTopologyBuilder("exclamation"); builder.addBolt(new ExclaimBolt(), 3); cluster.submitTopology("DRPCTest", conf, builder.createLocalTopology(drpc));log.debug("傳入參數返回的結果:"+drpc.execute("exclamation", "hello"));cluster.shutdown();drpc.shutdown();}}

運行結果