日志流量管理

FLUME参考案例：https://blog.csdn.net/weixin_43326910/article/details/111713326

日志数据原信息：https://blog.csdn.net/weixin_42229056/article/details/82775490

SQLyog：https://blog.csdn.net/Sunshine_liang1/article/details/84400820

一丶启动hadoop高可用集群

zkServer.sh start    (三台机器)
start-all.sh 第一台机器

二丶启动flume采集数据

flume-ng agent --conf conf/ --conf-file /export/lastwork/flume/Taildir-web.conf --name a1 -Dflume.root.logger=INFO,console

hdfs dfs -rm -r /flume

 三丶启动MYSQL服务

 1 systemctl start mysqld.service  

四、HIVE

 hive 

 drop database weblog; 

 drop database weblog cascade; 

 采集方案如下：

a1.sources=r1
#数据沉淀，可以有多个，中间用,隔开
a1.sinks=k1
#传输管道，一般只有一个
a1.channels=c1
#采集消息的类型
a1.sources.r1.type=TAILDIR
#检查点文件的路径
a1.sources.r1.positionFile=/var/log/flume/taildir_position.json
#监控该目录下的文件
a1.sources.r1.filegroups=f1 f2
a1.sources.r1.filegroups.f1=/var/log/test1/example.log
a1.sources.r1.filegroups.f2=/var/log/test2/.*log.*
#配置channel
a1.channels.c1.type=memory
a1.channels.c1.capacity=100
a1.channels.c1.transactionCapacity=100
a1.channels.c1.keep-alive=10
a1.channels.c1.byteCapacity=0
#配置sink
#4.配置并描述sink
a1.sinks.k1.type=hdfs              
#4.1. rollInterval、rollCount、rollSize 分别是每隔多少秒写一个文件、往文件中写入数据的最大次数、rollSize文件的大小。也就是达到任意一个条件hdfs上就会生成一个新文件，继续写入
a1.sinks.rollInterval=60
a1.sinks.rollCount=30000
a1.sinks.rollSize=10240
a1.sinks.idleTimeout=60 
a1.sinks.k1.hdfs.round=true
a1.sinks.k1.hdfs.roundValue=10 
a1.sinks.k1.hdfs.roundUnit=minuite
a1.sinks.k1.hdfs.useLocalTimeStamp=true
#沉淀的方式是控制台打印,这里是上传到hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.52.150:9000/flume/testLog.log
    
a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

 将文件从hadoop上复制下来

 hadoop fs -get /flume/testLog.log /export/lastwork/flumedata/ 

后面是自己虚拟机的路径

 

 

hive

 create database weblog; 

create table ods_weblog_origin(
valid string,
remote_addr string,
remote_user string,
time_local string,
request string,
status string,
body_bytes_sent string,
http_referer string,
http_user_agent string)
partitioned by (datestr string)
row format delimited
fields terminated by '\001';

 hadoop fs -mkdir -p /weblog/preprocessed; 

 hadoop fs -put part-m-00000 /weblog/preprocessed; 

在hive中导入数据

 load data inpath '/weblog/preprocessed/' overwrite into table ods_weblog_origin partition(datestr='20130918'); 

生成明细表

create table ods_weblog_detail(
valid string,
remote_addr string,
remote_user string,
time_local string,
daystr string,
timestr string,
month string,
day string,
hour string,
request string,
status string,
body_bytes_sent string,
http_referer string,
ref_host string,
ref_path string,
ref_query string,
ref_query_id string,
http_user_agent string
)
partitioned by(datestr string);

 创建临时中间表

create table t_ods_tmp_referurl as
SELECT a.*,b.*
FROM ods_weblog_origin a LATERAL VIEW
parse_url_tuple(regexp_replace(http_referer,"\"",""),
'HOST','PATH','QUERY','QUERY:id') b as host,path,query,query_id;

创建临时中间表解析时间字段

create table t_ods_tmp_detail as
select b.*,substring(time_local,0,10) as daystr,
substring(time_local,12) as tmstr,
substring(time_local,6,2) as month,
substring(time_local,9,2) as day,
substring(time_local,11,3) as hour
from t_ods_tmp_referurl b;

 

set hive.exec.dynamic.partition=true;

set hive.exec.dynamic.partition.mode=nonstrict; 

 

 

insert overwrite table ods_weblog_detail partition(datestr)
select distinct otd.valid,otd.remote_addr,otd.remote_user,
otd.time_local,otd.daystr,otd.tmstr,otd.month,otd.day,otd.hour,
otr.request,otr.status,otr.body_bytes_sent,
otr.http_referer,otr.host,otr.path,
otr.query,otr.query_id,otr.http_user_agent,otd.daystr
from t_ods_tmp_detail as otd,t_ods_tmp_referurl as otr
where otd.remote_addr=otr.remote_addr
and otd.time_local=otr.time_local
and otd.body_bytes_sent=otr.body_bytes_sent
and otd.request=otr.request;

 create table dw_pvs_everyday(pvs bigint,month string,day string); 

 

insert into table dw_pvs_everyday
select count(*) as pvs,owd.month as month,owd.day as day
from ods_weblog_detail owd
group by owd.month,owd.day;

 

create table dw_avgpv_user_everyday(
day string
avgpv string);

 

 

insert into table dw_avgpv_user_everyday
select '2013-09-18',sum(b.pvs)/count(b.remote_addr) from
(select remote_addr,count(1) as pvs from ods_weblog_detail where
datestr='2013-09-18' group by remote_addr) b;

 

 

在mysql里：

 create DATABASE sqoopdb; 

在sqoop安装节点

bin/sqoop export \
--connect jdbc:mysql://node01:3306/sqoopdb?useSSL=false \
--username root \
--password 123456 \
--table t_avgpv_num \
--columns "dateStr,avgPvNum" \
--fields-terminated-by '\001' \
--export-dir /user/hive/warehouse/weblog.db/dw_avgpv_user_everyday