Storm-wordcount实时统计单词次数
一、本地模式
1、WordCountSpout类
package com.demo.wc; import java.util.Map; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichSpout; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; /** * 需求：单词计数 hello world hello Beijing China * * 实现接口： IRichSpout IRichBolt * 继承抽象类：BaseRichSpout BaseRichBolt 常用*/ public class WordCountSpout extends BaseRichSpout { //定义收集器 private SpoutOutputCollector collector; //发送数据 @Override public void nextTuple() { //1.发送数据 到bolt collector.emit(new Values("I like China very much")); //2.设置延迟 try { Thread.sleep(500); } catch (InterruptedException e) { e.printStackTrace(); } } //创建收集器 @Override public void open(Map arg0, TopologyContext arg1, SpoutOutputCollector collector) { this.collector = collector; } //声明描述 @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { //起别名 declarer.declare(new Fields("wordcount")); } }
2、WordCountSplitBolt类
package com.demo.wc; import java.util.Map; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; public class WordCountSplitBolt extends BaseRichBolt { //数据继续发送到下一个bolt private OutputCollector collector; //业务逻辑 @Override public void execute(Tuple in) { //1.获取数据 String line = in.getStringByField("wordcount"); //2.切分数据 String[] fields = line.split(" "); //3.<单词,1> 发送出去 下一个bolt(累加求和) for (String w : fields) { collector.emit(new Values(w, 1)); } } //初始化 @Override public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { this.collector = collector; } //声明描述 @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word", "sum")); } }
3、WordCountBolt类
package com.demo.wc; import java.util.HashMap; import java.util.Map; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Tuple; public class WordCountBolt extends BaseRichBolt{ private Map<String, Integer> map = new HashMap<>(); //累加求和 @Override public void execute(Tuple in) { //1.获取数据 String word = in.getStringByField("word"); Integer sum = in.getIntegerByField("sum"); //2.业务处理 if (map.containsKey(word)) { //之前出现几次 Integer count = map.get(word); //已有的 map.put(word, count + sum); } else { map.put(word, sum); } //3.打印控制台 System.out.println(Thread.currentThread().getName() + "\t 单词为：" + word + "\t 当前已出现次数为：" + map.get(word)); } @Override public void prepare(Map arg0, TopologyContext arg1, OutputCollector arg2) { } @Override public void declareOutputFields(OutputFieldsDeclarer arg0) { } }
4、WordCountDriver类
package com.demo.wc; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.tuple.Fields; public class WordCountDriver { public static void main(String[] args) { //1.hadoop->Job storm->topology 创建拓扑 TopologyBuilder builder = new TopologyBuilder(); //2.指定设置 builder.setSpout("WordCountSpout", new WordCountSpout(), 1); builder.setBolt("WordCountSplitBolt", new WordCountSplitBolt(), 4).fieldsGrouping("WordCountSpout", new Fields("wordcount")); builder.setBolt("WordCountBolt", new WordCountBolt(), 2).fieldsGrouping("WordCountSplitBolt", new Fields("word")); //3.创建配置信息 Config conf = new Config(); //4.提交任务 LocalCluster localCluster = new LocalCluster(); localCluster.submitTopology("wordcounttopology", conf, builder.createTopology()); } }
5、直接运行（4）里面的main方法即可启动本地模式。
二、集群模式
前三个类和上面本地模式一样，第4个类WordCountDriver和本地模式有点区别
package com.demo.wc; import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.tuple.Fields; public class WordCountDriver { public static void main(String[] args) { //1.hadoop->Job storm->topology 创建拓扑 TopologyBuilder builder = new TopologyBuilder(); //2.指定设置 builder.setSpout("WordCountSpout", new WordCountSpout(), 1); builder.setBolt("WordCountSplitBolt", new WordCountSplitBolt(), 4).fieldsGrouping("WordCountSpout", new Fields("wordcount")); builder.setBolt("WordCountBolt", new WordCountBolt(), 2).fieldsGrouping("WordCountSplitBolt", new Fields("word")); //3.创建配置信息 Config conf = new Config(); //conf.setNumWorkers(10); //集群模式 try { StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } catch (Exception e) { e.printStackTrace(); } //4.提交任务 //LocalCluster localCluster = new LocalCluster(); //localCluster.submitTopology("wordcounttopology", conf, builder.createTopology()); } }
把程序打成jar包放在启动了Storm集群的机器里，在stormwordcount.jar所在目录下执行
storm jar stormwordcount.jar com.demo.wc.WordCountDriver wordcount01
即可启动程序。
三、并发度和分组策略
1、WordCountDriver_Shuffle类
package com.demo.wc; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.topology.TopologyBuilder; public class WordCountDriver_Shuffle { public static void main(String[] args) { //1.hadoop->Job storm->topology 创建拓扑 TopologyBuilder builder = new TopologyBuilder(); //2.指定设置 builder.setSpout("WordCountSpout", new WordCountSpout(), 2); builder.setBolt("WordCountSplitBolt", new WordCountSplitBolt(), 2).setNumTasks(4).shuffleGrouping("WordCountSpout"); builder.setBolt("WordCountBolt", new WordCountBolt(), 6).shuffleGrouping("WordCountSplitBolt"); //3.创建配置信息 Config conf = new Config(); //conf.setNumWorkers(2); //集群模式 // try { // StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); // } catch (Exception e) { // e.printStackTrace(); // } //4.提交任务 LocalCluster localCluster = new LocalCluster(); localCluster.submitTopology("wordcounttopology", conf, builder.createTopology()); } }
2、并发度与分组策略
