MapReduce平均值
package com.bw.hadoop;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Average {
/**
*
* @param args
* @throws IOException
* @throws ClassNotFoundException
* @throws InterruptedException
*
* 2017-8-12 10:13
* LPY
*
*/
@SuppressWarnings("unused")
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://192.168.0.117:9000");
config.set("yarn.resourcemanager.hostname", "192.168.0.117");
Job job = Job.getInstance(config);
job.setMapperClass(mapper.class);
job.setReducerClass(reducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job, new Path("/Avg"));
FileOutputFormat.setOutputPath(job, new Path("/Out"));
boolean b = job.waitForCompletion(true);
if(b){
System.out.println("Success");
}else{
System.out.println("Error");
}
}
public static class mapper extends Mapper<Object, Text, Text, Text>{
@Override
protected void map(Object key, Text value, Mapper<Object, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
String[] line = value.toString().split(" ",2);
context.write(new Text(line[0]), new Text(line[1]));
}
}
public static class reducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text text, Iterable<Text> iterable, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
float count = 0;
float sum = 0;
for(Text val: iterable){
sum += Float.parseFloat(val.toString());
count += 1.00f;
}
sum = sum / count;
context.write(text, new Text(String.valueOf(sum)));
}
}
}
浙公网安备 33010602011771号