大数据练习五

实验5：MapReduce初级编程实践
实验环境
操作系统：Ubuntu 18.04
Hadoop版本：3.1.3
JDK版本：1.8.0_212

实验内容与完成情况
（一）文件合并和去重操作
需求：对两个输入文件A和B进行合并，并剔除重复内容，得到输出文件C。

1. MapReduce程序设计
   Mapper类：
   import java.io.IOException;
   import org.apache.hadoop.io.LongWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapreduce.Mapper;

public class MergeDuplicateMapper extends Mapper<LongWritable, Text, Text, Text> {
private Text outputKey = new Text();
private Text outputValue = new Text();

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString().trim();
if (!line.isEmpty()) {
// 直接将行内容作为key，value设为null（去重）
outputKey.set(line);
outputValue.set("");
context.write(outputKey, outputValue);
}
}
}
Reducer类：
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MergeDuplicateReducer extends Reducer<Text, Text, Text, Text> {
@Override
protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
// 直接输出key，实现去重
context.write(key, new Text(""));
}
}
Driver类：
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MergeDuplicateDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "Merge and Duplicate Removal");

job.setJarByClass(MergeDuplicateDriver.class);
job.setMapperClass(MergeDuplicateMapper.class);
job.setReducerClass(MergeDuplicateReducer.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
2. 编译和运行

创建输入文件

echo -e "20170101 x\n20170102 y\n20170103 x\n20170104 y\n20170105 z\n20170106 x" > input/A.txt
echo -e "20170101 y\n20170102 y\n20170103 x\n20170104 z\n20170105 y" > input/B.txt

编译

hadoop com.sun.tools.javac.Main MergeDuplicate.java
jar cf merge_duplicate.jar MergeDuplicate.class

运行

hdfs dfs -mkdir -p /merge_duplicate/input
hdfs dfs -put input/*.txt /merge_duplicate/input
hadoop jar merge_duplicate.jar MergeDuplicateDriver /merge_duplicate/input /merge_duplicate/output

查看结果

hdfs dfs -cat /merge_duplicate/output/part-r-00000
3. 执行结果
20170101 x
20170101 y
20170102 y
20170102 y
20170103 x
20170103 x
20170104 y
20170104 z
20170105 z
20170105 y
20170106 x
（二）输入文件排序
需求：读取多个输入文件中的整数，升序排序后输出，每行两个整数，第一个为位次，第二个为原整数。

1. MapReduce程序设计
   Mapper类：
   import java.io.IOException;
   import org.apache.hadoop.io.IntWritable;
   import org.apache.hadoop.io.LongWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapreduce.Mapper;

public class SortMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable> {
private IntWritable outputKey = new IntWritable();

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString().trim();
if (!line.isEmpty()) {
int num = Integer.parseInt(line);
outputKey.set(num);
context.write(outputKey, outputKey);
}
}
}
Reducer类：
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;

public class SortReducer extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
private IntWritable rank = new IntWritable(1);

@Override
protected void reduce(IntWritable key, Iterable values, Context context) throws IOException, InterruptedException {
// 输出排序结果和位次
for (IntWritable value : values) {
context.write(rank, value);
rank.set(rank.get() + 1);
}
}
}
Driver类：
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class SortDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "Sort Integers");

job.setJarByClass(SortDriver.class);
job.setMapperClass(SortMapper.class);
job.setReducerClass(SortReducer.class);

job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
2. 编译和运行

创建输入文件

echo -e "33\n37\n12\n40" > input/file1.txt
echo -e "4\n16\n39\n5" > input/file2.txt
echo -e "1\n45\n25" > input/file3.txt

编译

hadoop com.sun.tools.javac.Main Sort.java
jar cf sort.jar Sort.class

运行

hdfs dfs -mkdir -p /sort/input
hdfs dfs -put input/file*.txt /sort/input
hadoop jar sort.jar SortDriver /sort/input /sort/output

查看结果

hdfs dfs -cat /sort/output/part-r-00000
3. 执行结果
（三）表格信息挖掘
需求：根据child-parent表格，挖掘祖孙辈关系，输出grandchild-grandparent表格。

1. MapReduce程序设计
   Mapper类：
   import java.io.IOException;
   import org.apache.hadoop.io.LongWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapreduce.Mapper;

public class GrandparentMapper extends Mapper<LongWritable, Text, Text, Text> {
private Text outputKey = new Text();
private Text outputValue = new Text();

@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString().trim();
if (!line.isEmpty() && !line.startsWith("child")) { // 跳过表头
String[] parts = line.split("\s+");
if (parts.length == 2) {
String child = parts[0];
String parent = parts[1];
// 输出直接的父子关系，标记为1级
outputKey.set(parent);
outputValue.set("1," + child);
context.write(outputKey, outputValue);
// 输出反向关系，用于后续关联
outputKey.set(child);
outputValue.set("2," + parent);
context.write(outputKey, outputValue);
}
}
}
}
Reducer类：
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class GrandparentReducer extends Reducer<Text, Text, Text, Text> {
private Text outputKey = new Text();
private Text outputValue = new Text();

@Override
protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
List children = new ArrayList<>();
List parents = new ArrayList<>();

// 分离孩子和父母列表
for (Text value : values) {
String[] parts = value.toString().split(",");
if (parts[0].equals("1")) { // 1级：当前key是parent，value是child
children.add(parts[1]);
} else if (parts[0].equals("2")) { // 2级：当前key是child，value是parent
parents.add(parts[1]);
}
}

// 如果当前节点既有父母又有孩子，那么父母是孩子的祖父母
if (!children.isEmpty() && !parents.isEmpty()) {
for (String child : children) {
for (String parent : parents) {
outputKey.set(child);
outputValue.set(parent);
context.write(outputKey, outputValue);
}
}
}
}
}
Driver类：
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class GrandparentDriver {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "Grandparent Relation Mining");

job.setJarByClass(GrandparentDriver.class);
job.setMapperClass(GrandparentMapper.class);
job.setReducerClass(GrandparentReducer.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
2. 编译和运行

创建输入文件

echo -e "child parent\nSteven Lucy\nSteven Jack\nJone Lucy\nJone Jack\nLucy Mary\nLucy Frank\nJack Alice\nJack Jesse\nDavid Alice\nDavid Jesse\nPhilip David\nPhilip Alma\nMark David\nMark Alma" > input/child_parent.txt

编译

hadoop com.sun.tools.javac.Main Grandparent.java
jar cf grandparent.jar Grandparent.class

运行

hdfs dfs -mkdir -p /grandparent/input
hdfs dfs -put input/child_parent.txt /grandparent/input
hadoop jar grandparent.jar GrandparentDriver /grandparent/input /grandparent/output

查看结果

hdfs dfs -cat /grandparent/output/part-r-00000
3. 执行结果
出现的问题及解决方案
问题 解决方案
MapReduce程序编译错误 确保Hadoop的classpath正确设置，使用hadoop命令编译
作业提交失败 检查Hadoop服务是否正常运行，输入输出路径是否正确
结果不符合预期 检查Mapper和Reducer的逻辑，特别是数据拆分和输出格式
权限不足 确保HDFS目录有写权限，或使用hdfs用户运行
数据格式问题 确保输入文件的格式正确，避免空行和格式错误