Hadoop电商日志数据分析(一)

项目要求

根据电商日志文件，分析：
1 . 统计页面浏览量（每行记录就是一次浏览）
2 . 统计各个省份的浏览量 （需要解析IP）
3 . 日志的ETL操作（ETL：数据从来源端经过抽取（Extract）、转换（Transform）、加载（Load）至目的端的过程）

为什么要ETL：没有必要解析出所有数据，只需要解析出有价值的字段即可。本项目中需要解析出：ip、url、pageId（topicId对应的页面Id）、country、province、city

提供一个数据集和四个工具类：

rackinfo_20130721.data

点击查看数据集示例

20939177966
http://www.yihaodian.com/1/?type=3&tracker_u=10781349260


3
YU7317CWPWJPCCKSJD83RZ35W6F1DGHT4P93




PPGDPBNYMTMMVQQSX13BVW4A6Z61BCFK
10781349260
\N
115.28.44.205

,unionKey:10781349260

2013-07-21 01:18:59
\N

1

\N
null
-20




Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Sicent; WoShiHoney.B; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Microsoft Windows Media Center PC 6.0)
Win32





北京市
2


北京市






\N
\N
\N
\N
2013-07-21
20966958616
http://www.1mall.com/item/11033978_4
http://www.1mall.com/item/11033980_4?ref=1_1_51_search.keyword_1

3
EUBCC9CG8G7G3Z331PWEV3UNWD7QZA1F3Q2Q
1


[{1=[{9784616=38251:1190}]}]
PPH24C5RHH3GE4RUY1UKQW84PVYBXZFT
7439964
\N
101.18.106.107

msessionid:CP23DVRKZGQ67MWQQGNHN5USRK2R9W8V,unionKey:7439964

2013-07-21 15:11:16
\N
http://www.2345.com/?kskypw
5
9784615
\N
4
-1




Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML
Win32





河北省
4




46




\N
\N
\N
\N
2013-07-21
20966773971
http://search.yihaodian.com/s2/c0-0/k%25E9%259B%25AA%25E7%25BA%25BA%2520t%25E6%2581%25A4/1/
http://www.yihaodian.com/4/?tracker_u=7439964&type=1

3
EUBCC9CG8G7G3Z331PWEV3UNWD7QZA1F3Q2Q




PPH24C5RHH3GE4RUY1UKQW84PVYBXZFT
7439964
\N
101.18.106.107

msessionid:WUEYCTYUHP7CUP4UD7Y2VHNZABPCYJ18,unionKey:7439964

2013-07-21 15:10:02
\N
http://www.2345.com/?kskypw
3

\N
4
-1




Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML
Win32
雪纺 t恤
1366
1
CatMenu_Keywords_100000003_18258

河北省
4
null



3




\N
\N
\N
\N
2013-07-21

日志字段说明

技术架构：

页面浏览量统计

1.页面编号的获取

统计页面的访问量：把符合规则的pageId获取到，然后进行统计即可。

• ContentUtils

package com.imooc.bigdata.hadoop.mr.project.utils;

import org.apache.commons.lang.StringUtils;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ContentUtils {
    public static String getPageId(String url){
        String pageId = "";
        if (StringUtils.isBlank(url)){
            return pageId;
        }

        //正则表达式
        Pattern pattern = Pattern.compile("topicId=[0-9]+");
        Matcher matcher = pattern.matcher(url);

        if (matcher.find()){
            pageId = matcher.group().split("topicId=")[1];
        }
        return pageId;
    }
}

• LogParseTest

package com.imooc.bigdata.hadoop.mr.project.utils;

import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.Map;

public class LogParseTest {

    LogParser logParser;

    @Before
    public void setUp(){
        logParser = new LogParser();
    }

    @After
    public void tearDown(){
        logParser = null;
    }

    @Test
    public void test01(){
        Map<String, String> map  = logParser.parse("20946835322\u0001http://www.yihaodian.com/1/?tracker_u=2225501&type=3\u0001http://www.baidu.com/s?wd=1%E5%8F%B7%E5%BA%97&rsv_bp=0&ch=&tn=baidu&bar=&rsv_spt=3&ie=utf-8&rsv_sug3=5&rsv_sug=0&rsv_sug1=4&rsv_sug4=313&inputT=4235\u00011号店\u00011\u0001SKAPHD3JZYH9EE9ACB1NGA9VDQHNJMX1NY9T\u0001\u0001\u0001\u0001\u0001PPG4SWG71358HGRJGQHQQBXY9GF96CVU\u00012225501\u0001\\N\u0001124.79.172.232\u0001\u0001msessionid:YR9H5YU7RZ8Y94EBJNZ2P5W8DT37Q9JH,unionKey:2225501\u0001\u00012013-07-21 09:30:01\u0001\\N\u0001http://www.baidu.com/s?wd=1%E5%8F%B7%E5%BA%97&rsv_bp=0&ch=&tn=baidu&bar=&rsv_spt=3&ie=utf-8&rsv_sug3=5&rsv_sug=0&rsv_sug1=4&rsv_sug4=313&inputT=4235\u00011\u0001\u0001\\N\u0001null\u0001-10\u0001\u0001\u0001\u0001\u0001Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MATP; Media Center PC 6.0; .NET4.0C; InfoPath.2; .NET4.0E)\u0001Win32\u0001\u0001\u0001\u0001\u0001\u0001上海市\u00011\u0001\u00012013-07-21 09:30:01\u0001上海市\u0001\u000166\u0001\u0001\u0001\u0001\u0001\\N\u0001\\N\u0001\\N\u0001\\N\u00012013-07-21\n");

        for(Map.Entry<String,String> entry : map.entrySet()) {
            System.out.println(entry.getKey() + " : " + entry.getValue());
        }
    }

    @Test
    public void test02(){
        String pageId = ContentUtils.getPageId("http://www.yihaodian.com/cms/view.do?topicId=19004");
        System.out.println(pageId);
    }
}

2.功能实现

点击查看代码

package com.imooc.bigdata.hadoop.mr.project.mr;

import com.imooc.bigdata.hadoop.mr.project.utils.GetPageId;
import com.imooc.bigdata.hadoop.mr.project.utils.LogParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.Map;

public class PageStatApp {

    // Driver端的代码：八股文
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();

        // 如果输出目录已经存在，则先删除
        FileSystem fileSystem = FileSystem.get(configuration);
        Path outputPath = new Path("output/v1/pagestat");
        if (fileSystem.exists(outputPath)) {
            fileSystem.delete(outputPath, true);
        }

        Job job = Job.getInstance(configuration);
        job.setJarByClass(PageStatApp.class);

        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);


        FileInputFormat.setInputPaths(job, new Path("/Users/rocky/data/trackinfo_20130721.data"));
        FileOutputFormat.setOutputPath(job, new Path("output/v1/pagestat"));

        job.waitForCompletion(true);
    }

    static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

        private LogParser parser;
        private LongWritable ONE = new LongWritable(1);

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            parser = new LogParser();
        }

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String log = value.toString();
            Map<String, String> logInfo = parser.parse(log);
            String pageId = GetPageId.getPageId(logInfo.get("url"));
            context.write(new Text(pageId), ONE);
        }
    }

    static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {

            long count = 0;
            for (LongWritable access : values) {
                count++;
            }
            context.write(key, new LongWritable(count));

        }
    }

}