hadoop中的helloword

设置classpath

设置classpath



实现WordCount
import java.io.*;
import org.apache.hadoop.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.jobcontrol.*;
import org.apache.hadoop.mapreduce.lib.map.*;
import org.apache.hadoop.mapreduce.lib.reduce.*;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
public class WordCount1
{
    public static class WordCountMapper extends Mapper<Object,Text,Text,IntWritable>
    {
         private final static IntWritable one = new IntWritable(1);
         private Text word = new Text();
         public void map(Object key,Text value,Context context) throws IOException,InterruptedException
         {
             String[] words = value.toString().split(" ");
             for(String str: words)
             {
                    word.set(str);
                    context.write(word,one);
             }
         }
    }
    public static class WordCountReducer extends Reducer<Text,IntWritable,Text,IntWritable>
    {
         public void reduce(Text key,Iterable<IntWritable>values,Context context) throws IOException,InterruptedException
         {
              int total =0;
              for(IntWritable val:values)
              {
                  total++;
              }
              context.write(key,new IntWritable(total));
         }
    }
    public static void main(String[] args) throws Exception {
          Configuration conf = new Configuration();
          //Job job = new Job(conf,"word count");
          Job job = Job.getInstance();
          job.setJobName("word count");
          job.setJarByClass(WordCount1.class);
          job.setMapperClass(WordCountMapper.class);
          job.setReducerClass(WordCountReducer.class);
          job.setOutputKeyClass(Text.class);
          job.setOutputValueClass(IntWritable.class);
          FileInputFormat.addInputPath(job,new Path(args[0]));
          FileOutputFormat.setOutputPath(job,new Path(args[1]));
          System.exit(job.waitForCompletion(true)?0:1);
    }
}
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.*;
import org.apache.hadoop.mapreduce.lib.output.*;
import org.apache.hadoop.mapreduce.lib.map.*;
import org.apache.hadoop.mapreduce.lib.reduce.*;
public class WordCountPredefined
{
      public static void main(String[] args) throws Exception
      {
        Configuration conf = new Configuration();
        //Job job = new Job(conf,"word count1");
        Job job = Job.getInstance();
        job.setJobName("word count1");
        job.setJarByClass(WordCountPredefined.class);
        job.setMapperClass(TokenCounterMapper.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));
        System.exit(job.waitForCompletion(true)?0:1);
      }
}

WordCount的简易方法

 

 

实现WordCount

1. import java.io.*;
2. import org.apache.hadoop.*;
3. import org.apache.hadoop.conf.Configuration;
4. import org.apache.hadoop.fs.Path;
5. import org.apache.hadoop.io.IntWritable;
6. import org.apache.hadoop.io.Text;
7. import org.apache.hadoop.mapreduce.*;
9. import org.apache.hadoop.mapreduce.lib.jobcontrol.*;
10. import org.apache.hadoop.mapreduce.lib.map.*;
11. import org.apache.hadoop.mapreduce.lib.reduce.*;
12. import org.apache.hadoop.mapreduce.lib.input.*;
13. import org.apache.hadoop.mapreduce.lib.output.*;
16. public class WordCount1
17. {
18. public static class WordCountMapper extends Mapper<Object,Text,Text,IntWritable>
19. {
20. private final static IntWritable one = new IntWritable(1);
21. private Text word = new Text();
23. public void map(Object key,Text value,Context context) throws IOException,InterruptedException
24. {
25. String[] words = value.toString().split(" ");
27. for(String str: words)
28. {
29. word.set(str);
30. context.write(word,one);
31. }
32. }
33. }
35. public static class WordCountReducer extends Reducer<Text,IntWritable,Text,IntWritable>
36. {
37. public void reduce(Text key,Iterable<IntWritable>values,Context context) throws IOException,InterruptedException
38. {
39. int total =0;
40. for(IntWritable val:values)
41. {
42. total++;
43. }
44. context.write(key,new IntWritable(total));
45. }
46. }
48. public static void main(String[] args) throws Exception {
49. Configuration conf = new Configuration();
50. //Job job = new Job(conf,"word count");
51. Job job = Job.getInstance();
52. job.setJobName("word count");
53. job.setJarByClass(WordCount1.class);
54. job.setMapperClass(WordCountMapper.class);
55. job.setReducerClass(WordCountReducer.class);
56. job.setOutputKeyClass(Text.class);
57. job.setOutputValueClass(IntWritable.class);
58. FileInputFormat.addInputPath(job,new Path(args[0]));
59. FileOutputFormat.setOutputPath(job,new Path(args[1]));
60. System.exit(job.waitForCompletion(true)?0:1);
61. }
62. }

1. import org.apache.hadoop.conf.Configuration;
2. import org.apache.hadoop.fs.Path;
3. import org.apache.hadoop.io.IntWritable;
4. import org.apache.hadoop.io.Text;
5. import org.apache.hadoop.mapreduce.Job;
6. import org.apache.hadoop.mapreduce.lib.input.*;
7. import org.apache.hadoop.mapreduce.lib.output.*;
8. import org.apache.hadoop.mapreduce.lib.map.*;
9. import org.apache.hadoop.mapreduce.lib.reduce.*;
11. public class WordCountPredefined
12. {
13. public static void main(String[] args) throws Exception
14. {
15. Configuration conf = new Configuration();
17. //Job job = new Job(conf,"word count1");
18. Job job = Job.getInstance();
19. job.setJobName("word count1");
21. job.setJarByClass(WordCountPredefined.class);
22. job.setMapperClass(TokenCounterMapper.class);
23. job.setReducerClass(IntSumReducer.class);
24. job.setOutputKeyClass(Text.class);
25. job.setOutputValueClass(IntWritable.class);
27. FileInputFormat.addInputPath(job,new Path(args[0]));
28. FileOutputFormat.setOutputPath(job,new Path(args[1]));
30. System.exit(job.waitForCompletion(true)?0:1);
31. }
32. }

WordCount的简易方法