MapReduce的CombineFileInputFormat使用 

  1 package com.mengyao.hadoop.mapreduce;
  2 
  3 import java.io.IOException;
  4 import java.util.Iterator;
  5 
  6 import org.apache.hadoop.conf.Configuration;
  7 import org.apache.hadoop.conf.Configured;
  8 import org.apache.hadoop.fs.Path;
  9 import org.apache.hadoop.io.LongWritable;
 10 import org.apache.hadoop.io.Text;
 11 import org.apache.hadoop.mapreduce.Counter;
 12 import org.apache.hadoop.mapreduce.Job;
 13 import org.apache.hadoop.mapreduce.Mapper;
 14 import org.apache.hadoop.mapreduce.Reducer;
 15 import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
 16 import org.apache.hadoop.mapreduce.lib.input.CombineTextInputFormat;
 17 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 18 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 19 import org.apache.hadoop.util.Tool;
 20 import org.apache.hadoop.util.ToolRunner;
 21 
 22 /**
 23  * 使用CombinerTextInputFormat可以把多个小文件打包到一个输入分片中使每个Mapper任务处理更多的数据。
 24  * 这样就避免了像FileInputFormat那样为每个文件（无论文件大小）都使用一个分片导致遇到小文件时会有很多的Mapper任务（每个Mapper任务处理的数据很小）。
 25  *
 26  * @author mengyao
 27  *
 28  */
 29 public class CombinerTextInputFormatApp extends Configured implements Tool {
 30 
 31     static class CombinerTextInputFormatMapper extends Mapper<LongWritable, Text, LongWritable, Text> {
 32         
 33         @Override
 34         protected void setup(Context context)
 35                 throws IOException, InterruptedException {
 36         }
 37         
 38         @Override
 39         protected void map(LongWritable key, Text value, Context context)
 40                 throws IOException, InterruptedException {
 41             CombineFileSplit cfs = (CombineFileSplit)context.getInputSplit();
 42             String fileA = cfs.getPath(0).getName();
 43             String fileB = cfs.getPath(1).getName();
 44             Counter counterA = context.getCounter(fileA, fileA);
 45             Counter counterB = context.getCounter(fileB, fileB);
 46             if (fileA.equals(counterA.getName())) {
 47                 counterA.increment(1L);
 48             }
 49             if (fileB.equals(counterB.getName())) {
 50                 counterB.increment(1L);
 51             }
 52             context.write(key,value);
 53         }
 54     }
 55     
 56     static class CombinerTextInputFormatReducer extends Reducer<LongWritable, Text, LongWritable, Text> {
 57         
 58         @Override
 59         protected void setup(Context context)
 60                 throws IOException, InterruptedException {
 61         }
 62         
 63         @Override
 64         protected void reduce(LongWritable key, Iterable<Text> value, Context context)
 65                 throws IOException, InterruptedException {
 66             Iterator<Text> iterator = value.iterator();
 67             while (iterator.hasNext()) {
 68                 context.write(key, iterator.next());
 69             }
 70         }
 71     }
 72 
 73     @Override
 74     public int run(String[] args) throws Exception {
 75         Job job = Job.getInstance(getConf(), CombinerTextInputFormatApp.class.getSimpleName());
 76         job.setJarByClass(CombinerTextInputFormatApp.class);
 77         
 78         job.setInputFormatClass(CombineTextInputFormat.class);
 79         FileInputFormat.addInputPath(job, new Path(args[0]));
 80         FileOutputFormat.setOutputPath(job, new Path(args[1]));
 81         
 82         job.setMapperClass(CombinerTextInputFormatMapper.class);
 83         job.setMapOutputKeyClass(LongWritable.class);
 84         job.setMapOutputValueClass(Text.class);
 85         
 86         job.setReducerClass(CombinerTextInputFormatReducer.class);
 87         job.setOutputKeyClass(LongWritable.class);
 88         job.setOutputValueClass(Text.class);
 89         
 90         return job.waitForCompletion(true)?0:1;
 91     }
 92     
 93     public static int createJob(String[] args) {
 94         Configuration conf = new Configuration();
 95         conf.set("dfs.datanode.socket.write.timeout", "7200000");
 96         conf.set("mapreduce.input.fileinputformat.split.minsize", "268435456");
 97         conf.set("mapreduce.input.fileinputformat.split.maxsize", "536870912");
 98         conf.set("mapreduce.job.jvm.numtasks", "-1");        
 99         conf.set("mapreduce.map.speculative", "false");        
100         conf.set("mapreduce.reduce.speculative", "false");    
101         conf.set("mapreduce.map.maxattempts", "4");            
102         conf.set("mapreduce.reduce.maxattempts", "4");        
103         conf.set("mapreduce.map.skip.maxrecords", "0");
104         int status = 0;
105         
106         try {
107             status = ToolRunner.run(conf, new CombinerTextInputFormatApp(), args);
108         } catch (Exception e) {
109             e.printStackTrace();
110         }
111         
112         return status;
113     }
114     
115     public static void main(String[] args) {
116         args = new String[]{"/mapreduces/*.txt", "/mapreduces/combinertextinputFormat"};
117         if (args.length!=2) {
118             System.out.println("Usage: "+CombinerTextInputFormatApp.class.getName()+" Input paramters <INPUT_PATH> <OUTPUT_PATH>");
119         } else {
120             int status = createJob(args);
121             System.exit(status);
122         }
123     }
124     
125 }

 

posted @ 2013-02-06 23:13  孟尧  阅读(625)  评论(1)    收藏  举报