统计被引用计数

View Code 
 1 import java.io.IOException;
 2 import java.util.Iterator;
 3 
 4 import org.apache.hadoop.conf.Configuration;
 5 import org.apache.hadoop.conf.Configured;
 6 import org.apache.hadoop.fs.Path;
 7 import org.apache.hadoop.io.IntWritable;
 8 import org.apache.hadoop.io.Text;
 9 import org.apache.hadoop.mapred.FileInputFormat;
10 import org.apache.hadoop.mapred.FileOutputFormat;
11 import org.apache.hadoop.mapred.JobClient;
12 import org.apache.hadoop.mapred.JobConf;
13 import org.apache.hadoop.mapred.KeyValueTextInputFormat;
14 import org.apache.hadoop.mapred.MapReduceBase;
15 import org.apache.hadoop.mapred.Mapper;
16 import org.apache.hadoop.mapred.OutputCollector;
17 import org.apache.hadoop.mapred.Reducer;
18 import org.apache.hadoop.mapred.Reporter;
19 import org.apache.hadoop.mapred.TextOutputFormat;
20 import org.apache.hadoop.util.Tool;
21 import org.apache.hadoop.util.ToolRunner;
22 
23 public class Slist_1 extends Configured implements Tool {
24     
25     public static class MapClass extends MapReduceBase
26         implements Mapper<Text, Text, IntWritable, IntWritable> {
27         
28         private final static IntWritable uno = new IntWritable(1);
29         private IntWritable citationCount = new IntWritable();
30         
31         public void map(Text key, Text value,
32                         OutputCollector<IntWritable, IntWritable> output,
33                         Reporter reporter) throws IOException {
34                         
35             citationCount.set(Integer.parseInt(value.toString()));
36             output.collect(citationCount, uno);
37         }
38     }
39     
40     public static class Reduce extends MapReduceBase
41         implements Reducer<IntWritable,IntWritable,IntWritable,IntWritable>
42     {
43         
44         public void reduce(IntWritable key, Iterator<IntWritable> values,
45                            OutputCollector<IntWritable, IntWritable>output,
46                            Reporter reporter) throws IOException {
47                            
48             int count = 0;
49             while (values.hasNext()) {
50                 count += values.next().get();
51             }
52             output.collect(key, new IntWritable(count));
53         }
54     }
55     
56     public int run(String[] args) throws Exception {
57         Configuration conf = getConf();
58         
59         JobConf job = new JobConf(conf, Slist_1.class);
60         
61         Path in = new Path(args[0]);
62         Path out = new Path(args[1]);
63         FileInputFormat.setInputPaths(job, in);
64         FileOutputFormat.setOutputPath(job, out);
65         
66         job.setJobName("Slist_1");
67         job.setMapperClass(MapClass.class);
68         job.setReducerClass(Reduce.class);
69         
70         job.setInputFormat(KeyValueTextInputFormat.class);
71         job.setOutputFormat(TextOutputFormat.class);
72         job.setOutputKeyClass(IntWritable.class);
73         job.setOutputValueClass(IntWritable.class);
74         job.set("key.value.separator.in.input.line", ",");
75         
76         JobClient.runJob(job);
77         
78         return 0;
79     }
80     
81     public static void main(String[] args) throws Exception { 
82         int res = ToolRunner.run(new Configuration(), 
83                                  new Slist_1(), 
84                                  args);
85         
86         System.exit(res);
87     }
88 }
posted @ 2012-09-12 12:45  aha~  阅读(166)  评论(0)    收藏  举报