Storm/JStorm之TopologyBuilder源码阅读
在Strom/JStorm中有一个类是特别重要的，主要用来构建Topology的，这个类就是TopologyBuilder.
咱先看一下简单的例子:
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("input", new RandomSentenceSpout(), 2); builder.setBolt("bolt_sentence", new SplitSentenceBolt(), 2) .shuffleGrouping("input"); // 本地模式：最主要用来调试用 LocalCluster cluster = new LocalCluster(); System.out.println("start wordcount"); cluster.submitTopology("word count", conf, builder.createTopology()); }
在上面的main方法里先创建TopologyBuilder对象,然后设置好已创建的Spout节点和Bolt节点，并用随机分组(shuffleGrouping)将Spout和Bolt节点连接起来形成Topology。
那TopologyBuilder是如何做的呢？请看下面TopologyBuilder源代码:
/** * TopologyBuilder是一个用于构建Topology的工具类 * */ public class TopologyBuilder { /** * 定义了类成员变量_bolts,用来存放IRichBolt类型的所有Bolt对象 */ private Map<String, IRichBolt> _bolts = new HashMap<String, IRichBolt>(); /** * 定义了类成员变量_spouts,用来存放IRichSpout类型的所有Spout对象 */ private Map<String, IRichSpout> _spouts = new HashMap<String, IRichSpout>(); /** * 定义了类成员变量_commons,存放了所有的Bolt和Spout对象 */ private Map<String, ComponentCommon> _commons = new HashMap<String, ComponentCommon>(); // private Map<String, Map<GlobalStreamId, Grouping>> _inputs = new HashMap<String, Map<GlobalStreamId, Grouping>>(); private Map<String, StateSpoutSpec> _stateSpouts = new HashMap<String, StateSpoutSpec>(); /** * 根据传入的Bolt和Spout对象构建StormTopology对象 * @return */ public StormTopology createTopology() { Map<String, Bolt> boltSpecs = new HashMap<String, Bolt>(); Map<String, SpoutSpec> spoutSpecs = new HashMap<String, SpoutSpec>(); for (String boltId : _bolts.keySet()) { //根据boltId从_bolts中获取到对应的bolt对象 IRichBolt bolt = _bolts.get(boltId); //设置对应ComponentCommon对象的streams(输出的字段列表以及是否是直接流)属性值 ComponentCommon common = getComponentCommon(boltId, bolt); /** * 先将Bolts对象序列化得到数组,再创建Bolt对象,所以所有在StormTopology中Bolts是对象序列化过后得到的字节数组. */ boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.javaSerialize(bolt)), common)); } for (String spoutId : _spouts.keySet()) { //根据spoutId从_spouts中获取到对应的spout对象 IRichSpout spout = _spouts.get(spoutId); //设置对应ComponentCommon对象的streams(输出的字段列表以及是否是直接流) ComponentCommon common = getComponentCommon(spoutId, spout); /** * 先将Spout对象序列化得到数组,再创建SpoutSpec对象,所以所有在StormTopology中Spouts是对象序列化过后得到的字节数组. */ spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.javaSerialize(spout)), common)); } //将上述所设置的所有组件都封装到StormTopology对象中,最后提交到集群中运行 return new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>()); } /** * 下面几个方法定义了setBolt方法以及它的重载方法 */ /** * 在这个topology中定义一个只有单线程并行度的新的bolt * 其它想要消耗这个bolt的输出的组件会引用这个id */ public BoltDeclarer setBolt(String id, IRichBolt bolt) { return setBolt(id, bolt, null); } /** * 为这个topology定义一个指定数量的并行度的bolt */ public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism_hint) { //检测传入的组件id是否唯一 validateUnusedId(id); //生成common对象 initCommon(id, bolt, parallelism_hint); _bolts.put(id, bolt); return new BoltGetter(id); } public BoltDeclarer setBolt(String id, IBasicBolt bolt) { return setBolt(id, bolt, null); } public BoltDeclarer setBolt(String id, IBasicBolt bolt, Number parallelism_hint) { /** * 该方法利用BasicBoltExecutor包装(封装)传入的IBasicBolt对象 * 在BasicBoltExecutor中实现了对消息的追踪 */ return setBolt(id, new BasicBoltExecutor(bolt), parallelism_hint); } /** * 下面几个方法定义了setSpout方法以及它的重载方法 */ public SpoutDeclarer setSpout(String id, IRichSpout spout) { return setSpout(id, spout, null); } public SpoutDeclarer setSpout(String id, IRichSpout spout, Number parallelism_hint) { //检测输入的id是否唯一,若已经存在将抛出异常 validateUnusedId(id); /** * 构建ComponentCommon对象并进行相对应的初始化,最后放入到_commons(在上述中已经定义) */ initCommon(id, spout, parallelism_hint); _spouts.put(id, spout); return new SpoutGetter(id); } public SpoutDeclarer setSpout(String id, IControlSpout spout) { return setSpout(id, spout, null); } public SpoutDeclarer setSpout(String id, IControlSpout spout, Number parallelism_hint) { return setSpout(id, new ControlSpoutExecutor(spout), parallelism_hint); } public BoltDeclarer setBolt(String id, IControlBolt bolt, Number parallelism_hint) { return setBolt(id, new ControlBoltExecutor(bolt), parallelism_hint); } public BoltDeclarer setBolt(String id, IControlBolt bolt) { return setBolt(id, bolt, null); } public void setStateSpout(String id, IRichStateSpout stateSpout) { setStateSpout(id, stateSpout, null); } public void setStateSpout(String id, IRichStateSpout stateSpout, Number parallelism_hint) { validateUnusedId(id); // TODO: finish } /** * 检测输入的id是否唯一 * @param id */ private void validateUnusedId(String id) { if (_bolts.containsKey(id)) { throw new IllegalArgumentException("Bolt has already been declared for id " + id); } if (_spouts.containsKey(id)) { throw new IllegalArgumentException("Spout has already been declared for id " + id); } if (_stateSpouts.containsKey(id)) { throw new IllegalArgumentException("State spout has already been declared for id " + id); } } private ComponentCommon getComponentCommon(String id, IComponent component) { ComponentCommon ret = new ComponentCommon(_commons.get(id)); OutputFieldsGetter getter = new OutputFieldsGetter(); component.declareOutputFields(getter); ret.set_streams(getter.getFieldsDeclaration()); return ret; } /** * 定义了initCommon方法，用来初始化变量CommonentCommon对象,并给类成员变量_commons赋值 * 初始化所做的工作:设置并行度还有一些其它配置 * @param id * @param component * @param parallelism */ private void initCommon(String id, IComponent component, Number parallelism) { ComponentCommon common = new ComponentCommon(); //设置消息流的来源及分组方式 common.set_inputs(new HashMap<GlobalStreamId, Grouping>()); if (parallelism != null) { //设置并行度 common.set_parallelism_hint(parallelism.intValue()); } else { //如果并行度没有手动设置则默认为1 common.set_parallelism_hint(1); } Map conf = component.getComponentConfiguration(); if (conf != null) //设置组件的配置参数 common.set_json_conf(JSONValue.toJSONString(conf)); _commons.put(id, common); } }
从上面TopologyBuilder的类中可以看到这个类提供了创建StormTopology的方法以及一些数据源节点和处理节点的相关设置的方法，
还有就是存储Bolt对象和Spout对象的方法，当然这里关于分组的代码没有写出来。事实上这个类就是用来设置Spout节点和Bolt节点，
并通过分组方式将Spout和Bolt节点连接起来形成拓扑结构的。
