WordCount例子

---

WordCountMapper.java

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import cn.itheima.bigdata.hadoop.WordCountRunner.MyselfCounter;

//在hadoop中，普通的java类不适合做网络序列化传输，hadoop对java的类型进行了封装，以便于利用hadoop的序列化框架进行序列化传输
public class WordCountMapper extends
  Mapper<LongWritable, Text, Text, LongWritable> {

 /**
  * map方法是每读一行调用一次
  */
 @Override
 protected void map(LongWritable key, Text value, Context context)
   throws IOException, InterruptedException {

  // 拿到一行的内容
  String line = value.toString();

  context.getCounter(MyselfCounter.MALFORMED).increment(1);
  context.getCounter("stringcounter", "TestCounter").increment(1);

  // 切分出一行中所有的单词
  String[] words = line.split(" ");
  // 输出<word,1>这种KV对
  for (String word : words) {
   // 遍历单词数组，一对一对地输出<hello,1> <tom,1> .......
   context.write(new Text(word), new LongWritable(1));

  }

 }

}

---

 

WordCountReducer.java

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
 /**
  * reduce方法是每获得一个<key,valueList>,执行一次
  */
 
 //key ： 某一个单词 ，比如  hello
 //values：  这个单词的所有v，  封装在一个迭代器中，可以理解为一个list{1,1,1,1.....}
 @Override
 protected void reduce(Text key, Iterable<LongWritable> values,Context context)
   throws IOException, InterruptedException {
  
  long count = 0;
  //遍历该key的valuelist，将所有value累加到计数器中去
  for(LongWritable value:values){
   count += value.get();
   
  }
    
  context.write(key, new LongWritable(count));
  
 }
 

}

---

 

WordCountCombiner.java

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountCombiner extends Reducer<Text, LongWritable, Text, LongWritable>{
 
 @Override
 protected void reduce(Text key, Iterable<LongWritable> values,Context context)
   throws IOException, InterruptedException {
  long count = 0;
  for(LongWritable value:values){
   
   count += value.get();
  }

  context.write(key, new LongWritable(count));
  
 }

}

---

public class WordCountRunner {

 enum MyselfCounter{
  MALFORMED,NORMAL
 }
 
 /**
  * mapreduce框架已经经历了两代API，旧API中的包名一般讲mapreduce简写成mapred
  * 直接用main函数来进行job提交，这是旧API的写法,在新API中，提交job的方法应该用ToolRunner中的run方法
  * @param args
  * @throws IOException
  * @throws ClassNotFoundException
  * @throws InterruptedException
  */
 public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
  Configuration conf = new Configuration();  
  //先构造一个用来提交我们的业务程序的一个信息封装对象
  Job job = Job.getInstance(conf); 
  //将我们的业务程序所在jar的路径封装到job中
  job.setJarByClass(WordCountRunner.class); 
  //指定本job所采用的mapper类
  job.setMapperClass(WordCountMapper.class);
  //指定本job所采用的reducer类
  job.setReducerClass(WordCountReducer.class); 
  //指定我们的mapper类输出的kv数据类型
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);  
  //指定我们的reducer类输出的kv数据类型
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class); 
  //指定数据读入的格式，可以不设置，因为默认就是用TextInputFormat.class
  job.setInputFormatClass(TextInputFormat.class);
  //指定数据输出的格式，可以不设置，因为默认就是用TextOutputFormat.class
  job.setOutputFormatClass(TextOutputFormat.class);
   //指定我们要处理的文件所在的路径
  FileInputFormat.setInputPaths(job, new Path("C:/testwordcount/data")); 
  //指定我们的输出结果文件所存放的路径
  FileOutputFormat.setOutputPath(job, new Path("C:/testwordcount/output2")); 
  job.waitForCompletion(true);
//  System.exit(job.waitForCompletion(true)?0:1);
//  System.out.println("-----------------------");
//  System.out.println(job.getCounters().findCounter(MyselfCounter.MALFORMED).getDisplayName());
//  System.out.println(job.getCounters().findCounter(MyselfCounter.MALFORMED).getValue());
//  System.out.println(job.getCounters().findCounter("stringcounter", "TestCounter").getDisplayName());
//  System.out.println(job.getCounters().findCounter("stringcounter", "TestCounter").getName());
//  System.out.println(job.getCounters().findCounter("stringcounter", "TestCounter").getValue());
 }

/**
 * 新API中对job提交类的建议写法
 */
public class WordCountDriver extends Configured implements Tool{ 
 /**
  * 在run方法中对job进行封装
  */
 @Override
 public int run(String[] args) throws Exception {  
  Configuration conf = new Configuration();  
  //先构造一个用来提交我们的业务程序的一个信息封装对象
  Job job = Job.getInstance(conf);  
  //将我们的业务程序所在jar的路径封装到job中
  job.setJarByClass(WordCountRunner.class);    
  //指定本job所采用的mapper类
  job.setMapperClass(WordCountMapper.class);
  //指定本job所采用的reducer类
  job.setReducerClass(WordCountReducer.class);  
  //指定本job所采用combiner组件
//  job.setCombinerClass(WordCountCombiner.class);
  job.setCombinerClass(WordCountReducer.class);  
  //指定我们的mapper类输出的kv数据类型
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);  
  //指定我们的reducer类输出的kv数据类型
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class);
  InputFormat input = null;
  //指定我们要处理的文件所在的路径
  FileInputFormat.setInputPaths(job, new Path("hdfs://weekend08:9000/wordcount/srcdata/"));  
  //指定我们的输出结果文件所存放的路径
  FileOutputFormat.setOutputPath(job, new Path("hdfs://weekend08:9000/wordcount/output2"));  
  return job.waitForCompletion(true)?0:1;
 }
 public static void main(String[] args) throws Exception {
  
  int res = ToolRunner.run(new Configuration(), new WordCountDriver(), args);
  System.exit(res);    
 }
}

---

 

2.流量例子