storm初步接触，一个单词计数的例子
我新建的maven工程
依赖如下：
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>0.9.1-incubating</version>
</dependency>
SentenceSpout（发送数据）代码如下：
package com.storm;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import backtype.storm.utils.Utils;
import java.util.Map;
/**
* Created by Administrator on 2017/6/15.
*/
public class SentenceSpout extends BaseRichSpout {
SpoutOutputCollector spoutOutputCollector;
private String [] sentences = {
"my dog has fleas",
"i like cold beverages",
"the dog ate my homework",
"don't have a cow man",
"i don't think i like fleas"
};
private int index =0;
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("sentence"));
}
@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
this.spoutOutputCollector = spoutOutputCollector;
}
@Override
public void nextTuple() {
this.spoutOutputCollector.emit(new Values(sentences[index]));
index++;
if(index>=sentences.length){
index = 0;
}
Utils.sleep(1);
}
}
SplitSentenceBolt（分割单词）：
package com.storm;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.Map;
/**
* Created by Administrator on 2017/6/15.
*/
public class SplitSentenceBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
}
@Override
public void execute(Tuple tuple) {
String sentence = tuple.getStringByField("sentence");
String [] words = sentence.split(" ");
for(String word : words){
this.collector.emit(new Values(word));
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word"));
}
}
WordCountBolt（单词计数）：
package com.storm;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;
/**
* Created by neil on 2017/6/15.
*/
public class WordCountBolt extends BaseRichBolt {
private OutputCollector collector;
private HashMap<String, Long> counts = null;
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
this.counts = new HashMap<String, Long>();
}
@Override
public void execute(Tuple tuple) {
String word = tuple.getStringByField("word");
Long count = this.counts.get(word);
if(count==null){
count = 0L;
}
count++;
this.counts.put(word, count);
this.collector.emit(new Values(word,count));
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word","count"));
}
}
ReportBolt（统计结果上报）：
package com.storm;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
import java.util.*;
/**
* Created by neil on 2017/6/15.
*/
public class ReportBolt extends BaseRichBolt {
private HashMap<String, Long> counts = null;
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.counts = new HashMap<String, Long>();
}
@Override
public void execute(Tuple tuple) {
String word = tuple.getStringByField("word");
Long count = tuple.getLongByField("count");
this.counts.put(word,count);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
//到这里结束了不往下走了
}
public void cleanup(){
System.out.println("--- PINAL COUNTS ---");
List<String> keys = new ArrayList<String>();
keys.addAll(this.counts.keySet());
Collections.sort(keys);
for(String key : keys){
System.out.println(key+":" +this.counts.get(key));
}
System.out.println("---------------------");
}
}
所有的bolt和spout都实现了IComponent接口，这个接口主要定义了declareOutputFields这个方法，storm的组件 通过这个方法告诉这一步（storm中的每一步都能看做数据的处理，
都有收上级数据和向下级发数据的处理过程。注：源头没有上级不用收。结尾没有下级不用发）会发射什么数据给下一步。
Ispout是spout类实现的接口，其主要方法是open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector)
map包含了storm配置信息（大家可以在open中通过调用map相关属性方法查看一下，不一一罗列了），TopologyContext提供了拓扑中的组件信息。
SpoutOutputCollector对象提供了发射元组（tuple）的方法。open主要做一些初始化的事情。
IBolt是bolt类实现的接口主要定义了perpare方法和Ispout的作用差不多，主要是做一些初始化的工作。提供发射元组方法的是OutputCollector对象。
spout和bolt主要的工作流程就是准备（open或者prepare）做处理（nextTuple或者excute）发射处理过后的结果。
WordCountTopology（单词技术topology）：
package com.storm;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
import backtype.storm.utils.Utils;
/**
* Created by neil on 2017/6/15.
*/
public class WordCountTopology {
private static final String SENTENCE_SPOUT_ID = "sentence-spout";
private static final String SPLIT_BOLT_ID = "split-bolt";
private static final String COUNT_BOLT_ID = "count-bolt";
private static final String REPORT_BOLT_ID = "report-bolt";
private static final String TOPOLOGY_NAME = "word-count-topology";
public static void main(String [] args) throws Exception{
SentenceSpout spout = new SentenceSpout();
SplitSentenceBolt splitBolt = new SplitSentenceBolt();
WordCountBolt countBolt = new WordCountBolt();
ReportBolt reportBolt = new ReportBolt();
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SENTENCE_SPOUT_ID, spout);
builder.setBolt(SPLIT_BOLT_ID,splitBolt).shuffleGrouping(SENTENCE_SPOUT_ID);//随机分发sentence_spout发送的单词
builder.setBolt(COUNT_BOLT_ID,countBolt).fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));//只接受同一个单词
builder.setBolt(REPORT_BOLT_ID,reportBolt).globalGrouping(COUNT_BOLT_ID);//所有的结果汇总到这里
Config config = new Config();
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(TOPOLOGY_NAME, config,builder.createTopology());
Utils.sleep(10000);
cluster.killTopology(TOPOLOGY_NAME);
cluster.shutdown();
}
}
以上代码基本上是从书上copy的 做了修改确保了能跑通：
storm被称作流式计算 从代码也可以看到了。从发送数据（句子）到分割单词到单词计数再到汇总报告，整个过程可以看成一个数据流的处理。
而WordCountTopology里面定义了数据是如何从源头流向末尾的即代码中的：
builder.setSpout(SENTENCE_SPOUT_ID, spout);//源头
builder.setBolt(SPLIT_BOLT_ID,splitBolt).shuffleGrouping(SENTENCE_SPOUT_ID);//接受源头的数据并做分割加工
builder.setBolt(COUNT_BOLT_ID,countBolt).fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));//接受分割好的数据并统计
builder.setBolt(REPORT_BOLT_ID,reportBolt).globalGrouping(COUNT_BOLT_ID);//把上一步统计好的数据汇总
浙公网安备 33010602011771号