【hadoop】MapReduce简单案例：WordCount

文章主要是介绍hadoop的mapreduce的一个小案例，主要是统计数据文本word.txt中每个单词出现的次数。

1、启动hadoop的dfs、yarn；

2、上传word.txt文件到hadoop的dfs中,可以使用命令或者用javaAPI上传

　命令上传 ,在当前文件目录下输入： hadoop fs -put word.txt /word.txt；

　　word.txt内容随意，博主的是:

3、打开eclipse，新建一个java项目，导入相关jar包（在hadoop下的的share目录中）

主要有这三个类：

1)WCMapper.java

package com.lxq.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * Created by Administrator on 2015/6/14.
 */
public class WCMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //读取数据
        String line = value.toString();
        //切分数据
        String[] words = line.split(" ");
        //遍历
        for (String word : words) {
            //每个单词作为key，记录1
            context.write(new Text(word),new LongWritable(1));
        }

    }
}

2)WCReducer

package com.lxq.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 * Created by Administrator on 2015/6/14.
 */
public class WCReducer extends Reducer<Text,LongWritable,Text,LongWritable> {

    @Override
    protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
        //接受数据
        //统计
        long counter = 0;
        for (LongWritable value : values) {
            counter += value.get();
        }
        //输出
        context.write(key,new LongWritable(counter));
    }

}

3)WordCount

package com.lxq.mapreduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * Created by Administrator on 2015/6/14.
 */
public class WordCount {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        //构建Job对象
        Job job = Job.getInstance(new Configuration());

        //设置main方法所在的类
        job.setJarByClass(WordCount.class);

        //设置Mapper相关属性
        job.setMapperClass(WCMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        //设置读取源文件的路径，这里应该是hdfs的文件路径
        FileInputFormat.setInputPaths(job,new Path("/word.txt"));

        //设置Reducer相关属性
        job.setReducerClass(WCReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        //设置计算结果的输出路径
        FileOutputFormat.setOutputPath(job,new Path("/wcout"));

        //提交任务 true 打印进度 false 不打印
        job.waitForCompletion(true);

    }

}