流量汇总例子

FlowBean.java

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;

/**
 * 在hadoop中，自定义的bean如果需要进行网络传输，必须实现hadoop的序列化框架，就是实现接口Writable
 * hadoop自带的序列化机制相比Serializable来说，传输的信息更加精简
 * 在hadoop项目的规划中，以后的序列化机制可能采用Avro框架（可以跨语言）
 * @author duanhaitao@itcast.cn
 *
 */

// WritableComparable<T> extends Writable, Comparable<T>
//但是在我们的自定义bean里面不能自己写成implements Writable,Comparable<T>,会报异常
public class FlowBean implements WritableComparable<FlowBean>{
 
 private String phoneNbr;
 private long up_flow;
 private long d_flow;
 private long sum_flow;
 
 
 //因为这个bean在反序列化时需要被反射出实例，就需要一个无参构造函数
 public FlowBean(){}
 
 public FlowBean(long up_flow,long d_flow){
  this.up_flow = up_flow;
  this.d_flow = d_flow;
  this.sum_flow = up_flow + d_flow;
 }
 
 
 public void set(String phoneNbr,long up_flow,long d_flow){
  this.phoneNbr = phoneNbr;
  this.up_flow = up_flow;
  this.d_flow = d_flow;
  this.sum_flow = up_flow + d_flow;
 }
 
 
 
 public long getUp_flow() {
  return up_flow;
 }
 public void setUp_flow(long up_flow) {
  this.up_flow = up_flow;
 }
 public long getD_flow() {
  return d_flow;
 }
 public void setD_flow(long d_flow) {
  this.d_flow = d_flow;
 }
 public long getSum_flow() {
  return sum_flow;
 }
 public void setSum_flow(long sum_flow) {
  this.sum_flow = sum_flow;
 }

 
 public String getPhoneNbr() {
  return phoneNbr;
 }

 public void setPhoneNbr(String phoneNbr) {
  this.phoneNbr = phoneNbr;
 }

 /**
  * 将对象中的信息序列化写入输出流
  */
 @Override
 public void write(DataOutput out) throws IOException {
  
  out.writeUTF(phoneNbr);
  out.writeLong(up_flow);
  out.writeLong(d_flow);
  out.writeLong(sum_flow);
  
 }

 /**
  * 从数据流中反序列化出各个字段，读的顺序要与序列化时写入的顺序一致
  */
 @Override
 public void readFields(DataInput in) throws IOException {

  phoneNbr = in.readUTF();
  up_flow = in.readLong();
  d_flow = in.readLong();
  sum_flow = in.readLong();
  
 }
 
 @Override
 public String toString() {
  
  return up_flow + "\t" + d_flow + "\t" + sum_flow;
 }

 @Override
 public int compareTo(FlowBean o) {
  
  return this.sum_flow > o.getSum_flow()?-1:1;
 }
 
 
}

---

public class FlowStatisticMapper extends Mapper<LongWritable, Text, Text, FlowBean>{
  
  
  @Override
  protected void map(LongWritable key, Text value,Context context)
    throws IOException, InterruptedException {

   //自定义一个计数器用来记录不合规的输入数据行数
   Counter lineErrCounter = context.getCounter("Malformed", "MalformedLine");
   
   //拿到一行的内容
   String line = value.toString();
   
   //切分出各个字段
   String[] fields = StringUtils.split(line,"\t");
   try{
   //取出上行流量和下行流量
   long up_flow = Long.parseLong(fields[fields.length-3]);
   long d_flow = Long.parseLong(fields[fields.length-2]);
   FlowBean bean = new FlowBean(up_flow,d_flow);
   
   //取出手机号
   String phoneNbr = fields[1];
   //为bean加入手机号值，以免序列化时出现空指针
   bean.setPhoneNbr(phoneNbr);
   
   context.write(new Text(phoneNbr),bean);
   }catch(Exception e){
    e.printStackTrace();
    lineErrCounter.increment(1);
    System.out.println("Exception occured in the mapper..........");
   }
  }
 }

---

public class FlowStatisticReducer extends Reducer<Text, FlowBean, Text, FlowBean>{
  
  
  @Override
  protected void reduce(Text key, Iterable<FlowBean> values,Context context)
    throws IOException, InterruptedException {

   long up_sum = 0;
   long d_sum = 0;
   for(FlowBean bean: values){
    up_sum += bean.getUp_flow();
    d_sum += bean.getD_flow();
   }
  
   FlowBean bean = new FlowBean(up_sum, d_sum);
   
   context.write(key, bean);
  
  }
  
 }

---

public class ProvincialPartitioner<KEY, VALUE> extends Partitioner<KEY, VALUE>{
 private static HashMap<String, Integer> areaMap = new HashMap<String, Integer>();
 
 //为了性能考虑，不能频繁查询外部数据库，而应该在任务启动之初一次性加载到内存中，然后广播给各个节点
 static{
  areaMap.put("135", 0);
  areaMap.put("136", 1);
  areaMap.put("137", 2);
  areaMap.put("139", 3);
  areaMap.put("159", 4);
 }
 
 
 @Override
 public int getPartition(KEY key, VALUE value, int numPartitions) {
  
  Integer provinceCode = areaMap.get(key.toString().substring(0, 3));
  
  return provinceCode==null?5:provinceCode;
 }

}

---

/**
 * 对流量日志进行按用户汇总统计
 * 如果要将结果按照手机号所属省份分文件输出，就要自定义一个partitioner，然后还要控制reduce task数量
 * @author duanhaitao@itcast.cn
 *
 */
public class FlowStatistic {

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
  
  Configuration conf = new Configuration();
  Job flowStatisticJob = Job.getInstance(conf);
  
  flowStatisticJob.setJarByClass(FlowStatistic.class);
  
  flowStatisticJob.setMapperClass(FlowStatisticMapper.class);
  flowStatisticJob.setReducerClass(FlowStatisticReducer.class);
  
  flowStatisticJob.setOutputKeyClass(Text.class);
  flowStatisticJob.setOutputValueClass(FlowBean.class);
  
  
  //指定reduce task的数量
  //partition的个数应该与reducetask数量保持一致
  //如果  reducetask数量 > partition ,则会产生多余的空结果文件
  //如果  reducetask数量 < partition ,则会抛出异常
  //如果  reducetask数量 < partition  &&   reducetask数量=1,也能正常运行，但是所有的kv都到了这一个reducer里面
  
  flowStatisticJob.setNumReduceTasks(5);
  //指定shuffle时使用partitioner类
  flowStatisticJob.setPartitionerClass(ProvincialPartitioner.class);
  
  
  FileInputFormat.setInputPaths(flowStatisticJob, new Path(args[0]));
  FileOutputFormat.setOutputPath(flowStatisticJob, new Path(args[1]));
  
  
  boolean res = flowStatisticJob.waitForCompletion(true);
  System.exit(res?0:1);
  
 }
 
}

---

 

2.将上面的结果，进行排序

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import cn.itheima.bigdata.hadoop.flow.FlowBean;

public class FlowSort {

 public static class FlowSortMapper extends Mapper<LongWritable, Text, FlowBean, NullWritable>{
  
  private FlowBean bean = new FlowBean();
  
  @Override
  protected void map(LongWritable key, Text value,Context context)
    throws IOException, InterruptedException {
   String line = value.toString();
   String[] fields = StringUtils.split(line,"\t");
   String phoneNbr = fields[0];
   long up_flow = Long.parseLong(fields[1]);
   long d_flow = Long.parseLong(fields[2]);
   bean.set(phoneNbr, up_flow, d_flow);
   context.write(bean,NullWritable.get());
   
  }
  
 }
 
 public static class FlowSortReducer extends Reducer<FlowBean, NullWritable, Text, FlowBean>{
  
  @Override
  protected void reduce(FlowBean bean, Iterable<NullWritable> values,Context context)
    throws IOException, InterruptedException {
   context.write(new Text(bean.getPhoneNbr()),bean);
  }
  
  
 }
 
 
 public static void main(String[] args) throws Exception {
  Configuration conf = new Configuration();
  Job job = Job.getInstance(conf);
  
  job.setJarByClass(FlowSort.class);
  
  job.setMapperClass(FlowSortMapper.class);
  job.setReducerClass(FlowSortReducer.class);
  
  job.setMapOutputKeyClass(FlowBean.class);
  job.setMapOutputValueClass(NullWritable.class);
  
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(FlowBean.class);
  
  FileInputFormat.setInputPaths(job, new Path(args[0]));
  FileOutputFormat.setOutputPath(job, new Path(args[1]));
  
  boolean res = job.waitForCompletion(true);
  System.exit(res?0:1);
 }
 
 
}