Hadoop生产集群的监视——计数器
可以在Hadoop作业中插桩计数器来分析其整体运作。在程序中定义不同的计数器，分别累计特定事件的发生次数。对于来自同一个作业所有任务的相同计数器，Hadoop会自动对它们进行求和， 以反映整个作业的情况。这些计数器的数值会在JobTracker的Web用户界面中与Hadoop的内部计数器一起显示。
计数器的典型应用是用来跟踪不同的输入记录类型，特别是跟踪“坏”记录。例如，我们得到的数据集格式为（只显示一部分）：
"PATENT","GYEAR","GDATE","APPYEAR","COUNTRY","POSTATE","ASSIGNEE","ASSCODE","CLAIMS","NCLASS","CAT","SUBCAT","CMADE","CRECEIVE","RATIOCIT","GENERAL","ORIGINAL","FWDAPLAG","BCKGTLAG","SELFCTUB","SELFCTLB","SECDUPBD","SECDLWBD" 3070801,1963,1096,,"BE","",,1,,269,6,69,,1,,0,,,,,,, 3070802,1963,1096,,"US","TX",,1,,2,6,63,,0,,,,,,,,, 3070803,1963,1096,,"US","IL",,1,,2,6,63,,9,,0.3704,,,,,,, 3070804,1963,1096,,"US","OH",,1,,2,6,63,,3,,0.6667,,,,,,, 3070805,1963,1096,,"US","CA",,1,,2,6,63,,1,,0,,,,,,, 3070806,1963,1096,,"US","PA",,1,,2,6,63,,0,,,,,,,,, 3070807,1963,1096,,"US","OH",,1,,623,3,39,,3,,0.4444,,,,,,, 3070808,1963,1096,,"US","IA",,1,,623,3,39,,4,,0.375,,,,,,, 3070809,1963,1096,,"US","AZ",,1,,4,6,65,,0,,,,,,,,, 3070810,1963,1096,,"US","IL",,1,,4,6,65,,3,,0.4444,,,,,,, 3070811,1963,1096,,"US","CA",,1,,4,6,65,,8,,0,,,,,,, 3070812,1963,1096,,"US","LA",,1,,4,6,65,,3,,0.4444,,,,,,, 3070813,1963,1096,,"US","NY",,1,,5,6,65,,2,,0,,,,,,, 3070814,1963,1096,,"US","MN",,2,,267,5,59,,2,,0.5,,,,,,, 3070815,1963,1096,,"US","CO",,1,,7,5,59,,1,,0,,,,,,, 3070816,1963,1096,,"US","OK",,1,,114,5,55,,4,,0,,,,,,, 3070817,1963,1096,,"US","RI",,2,,114,5,55,,5,,0.64,,,,,,, 3070818,1963,1096,,"US","IN",,1,,441,6,69,,4,,0.625,,,,,,, 3070819,1963,1096,,"US","TN",,4,,12,6,63,,0,,,,,,,,, 3070820,1963,1096,,"GB","",,2,,12,6,63,,0,,,,,,,,, 3070821,1963,1096,,"US","IL",,2,,15,6,69,,1,,0,,,,,,, 3070822,1963,1096,,"US","NY",,2,,401,1,12,,4,,0.375,,,,,,, 3070823,1963,1096,,"US","MI",,1,,401,1,12,,8,,0.6563,,,,,,, 3070824,1963,1096,,"US","IL",,1,,401,1,12,,5,,0.48,,,,,,, 3070825,1963,1096,,"US","IL",,1,,401,1,12,,7,,0.6531,,,,,,, 3070826,1963,1096,,"US","IA",,1,,401,1,12,,1,,0,,,,,,, 3070827,1963,1096,,"US","CA",,4,,401,1,12,,2,,0.5,,,,,,, 3070828,1963,1096,,"US","CT",,2,,16,5,59,,4,,0.625,,,,,,, 3070829,1963,1096,,"FR","",,3,,16,5,59,,5,,0.48,,,,,,, 3070830,1963,1096,,"US","NH",,2,,16,5,59,,0,,,,,,,,, 3070831,1963,1096,,"US","CT",,2,,16,5,59,,0,,,,,,,,, 3070832,1963,1096,,"US","LA",,2,,452,6,61,,1,,0,,,,,,, 3070833,1963,1096,,"US","LA",,1,,452,6,61,,5,,0,,,,,,, 3070834,1963,1096,,"US","FL",,1,,452,6,61,,3,,0.4444,,,,,,, 3070835,1963,1096,,"US","IL",,2,,264,5,51,,5,,0.64,,,,,,, 3070836,1963,1096,,"US","OK",,2,,264,5,51,,24,,0.7569,,,,,,, 3070837,1963,1096,,"CH","",,3,,264,5,51,,7,,0.6122,,,,,,, 3070838,1963,1096,,"CH","",,5,,425,5,51,,5,,0.48,,,,,,, 3070839,1963,1096,,"US","TN",,2,,425,5,51,,8,,0.4063,,,,,,, 3070840,1963,1096,,"GB","",,3,,425,5,51,,6,,0.7778,,,,,,, 3070841,1963,1096,,"US","OH",,2,,264,5,51,,6,,0.8333,,,,,,, 3070842,1963,1096,,"US","TX",,1,,425,5,51,,1,,0,,,,,,, 3070843,1963,1096,,"US","NY",,2,,425,5,51,,1,,0,,,,,,, 3070844,1963,1096,,"US","OH",,2,,425,5,51,,2,,0,,,,,,, 3070845,1963,1096,,"US","IL",,1,,52,6,69,,3,,0,,,,,,, 3070846,1963,1096,,"US","NY",,2,,425,5,51,,9,,0.7407,,,,,,,
我们想要计算每个国家专利声明的平均数，但是在许多记录中没有声明数。我们的程序会忽略这些记录，知道被忽略记录的数量是有用的。除了满足我们的好奇心，这种插桩让我们理解程序的操作并对其正确性做一些检查。
通过Reporter.incrCounter( )方法来使用计数器。Reporter对象被传递给map( )和reduce( )方法。以计数器名以及增量为参数来调用incrCounter( ) 。每个不同的事件都有一个独立命名的计数器。当用一个新的计数器名来调用incrCounter( )，这个计数器会被初始化并进行值的累加。
Reporter.incrCounter( )方法有两种签名：
public void incrCounter(String group, String counter, long amount) public void incrCounter(Enum key, long amount)
如下是使用了计数器之后的计算每个国家专利声明平均数的代码段：
package hadoop.in.action; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapred.TextOutputFormat; public class AverageByAttribute { public static class MapClass extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text> { static enum ClaimsCounters { MISSING, QUOTED }; private Text k = new Text(); private Text v = new Text(); @Override public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { String[] fields = value.toString().split(",", -1); String country = fields[4]; String numClaims = fields[8]; if (numClaims.length() == 0) { reporter.incrCounter(ClaimsCounters.MISSING, 1); } else { if (numClaims.startsWith("\"")) { reporter.incrCounter(ClaimsCounters.QUOTED, 1); } else { k.set(country); v.set(numClaims + ",1"); output.collect(k, v); } } } } public static class CombineClass extends MapReduceBase implements Reducer<Text, Text, Text, Text> { private Text v = new Text(); @Override public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { int count = 0; double sum = 0; while (values.hasNext()) { String[] fields = values.next().toString().split(","); sum += Double.parseDouble(fields[0]); count += Integer.parseInt(fields[1]); v.set(sum + "," + count); output.collect(key, v); } } } public static class ReduceClass extends MapReduceBase implements Reducer<Text, Text, Text, DoubleWritable> { private DoubleWritable v = new DoubleWritable(); @Override public void reduce(Text key, Iterator<Text> values, OutputCollector<Text, DoubleWritable> output, Reporter reporter) throws IOException { int count = 0; double sum = 0; while (values.hasNext()) { String[] fields = values.next().toString().split(","); sum += Double.parseDouble(fields[0]); count += Integer.parseInt(fields[1]); } v.set((double) sum / count); output.collect(key, v); } } public static void run() throws IOException { Configuration configuration = new Configuration(); JobConf jobConf = new JobConf(configuration, AverageByAttribute.class); String input = "hdfs://localhost:9000/user/hadoop/input/apat63_99.txt"; String output = "hdfs://localhost:9000/user/hadoop/output"; // HDFSDao hdfsDao = new HDFSDao(configuration); // hdfsDao.rmr(output); FileInputFormat.setInputPaths(jobConf, new Path(input)); FileOutputFormat.setOutputPath(jobConf, new Path(output)); jobConf.setInputFormat(TextInputFormat.class); jobConf.setOutputFormat(TextOutputFormat.class); jobConf.setMapOutputKeyClass(Text.class); jobConf.setMapOutputValueClass(Text.class); jobConf.setOutputKeyClass(Text.class); jobConf.setOutputValueClass(DoubleWritable.class); jobConf.setMapperClass(MapClass.class); jobConf.setCombinerClass(CombineClass.class); jobConf.setReducerClass(ReduceClass.class); RunningJob runningJob = JobClient.runJob(jobConf); while (!runningJob.isComplete()) { runningJob.waitForCompletion(); } } public static void main(String[] args) throws IOException { run(); } }
程序运行后，可以看到定义的计数器和Hadoop内部的计数器都被显示在JobTracker的Web用户界面中：
