flume 拦截器使用
拦截器使用:
使用目的:
Flume中的拦截器(interceptor),用户Source读取events发送到Sink的时候,在events header中加入一些有用的信息,或者对events的内容进行过滤,完成初步的数据清洗。
拦截器的使用方式:
通过配置文件
如果使用了flume集群,需要将该配置文件,发送到每个flume的conf目录下
拦截器加载方式:
拦截器打包之后,放入Flume的lib文件夹下面,flume会根据配置文件自动调用之(只需要单独包,不需要将依赖的包上传)。
如果使用了flume集群,需要将该jar包,发送到每个flume的lib目录下
附:拦截器使用的配置文件
a1.sources=r1
a1.channels=c1 c2
# configure source
a1.sources.r1.type = TAILDIR
a1.sources.r1.positionFile = /opt/module/flume/test/log_position.json
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1 = /tmp/logs/app.+
a1.sources.r1.fileHeader = true #flume数据的header将来会用到,如写header数据
a1.sources.r1.channels = c1 c2
#interceptor
a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type = com.flume.interceptor.ETLInterceptor$Builder #指定拦截器,$Builder表示加载方式
a1.sources.r1.interceptors.i2.type = com.flume.interceptor.TypeInterceptor$Builder
a1.sources.r1.selector.type = multiplexing #根据sourcer和sinke关系发送,
a1.sources.r1.selector.header = topic #以下三行,组成关系
a1.sources.r1.selector.mapping.topic_start = c1 # c1接收的数据是 header中 {"topic": "topic_start"}
a1.sources.r1.selector.mapping.topic_event = c2 # c2接收的数据是header中 {"topic": "topic_event"}
# configure channel
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.channels.c1.kafka.topic = topic_start
a1.channels.c1.parseAsFlumeEvent = false #不在数据的前面,加前缀。如果数据为data,加后就是 {"topic_start" : data}
a1.channels.c1.kafka.consumer.group.id = flume-consumer
a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c2.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.channels.c2.kafka.topic = topic_event
a1.channels.c2.parseAsFlumeEvent = false
a1.channels.c2.kafka.consumer.group.id = flume-consumer
拦截器实现:
package com.myflume;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import com.myflume.interceptor.LogUtils;
public class ETLInterceptor implements Interceptor {
public void initialize() {
// TODO Auto-generated method stub
}
public Event intercept(Event event) {
// TODO Auto-generated method stub
// 1- 获取数据
// 单个Event处理
byte[] body = event.getBody();
String log = new String(body, Charset.forName("UTF-8"));
// 2 校验 启动日志(JSON)
if (log.contains("start")) {
if(LogUtils.valuateStart(log)) {
return event;
}
}else {
if(LogUtils.valuateEvent(log)) {
return event;
}
}
return null;
}
public List<Event> intercept(List<Event> events) {
// TODO Auto-generated method stub
ArrayList<Event> interceptors = new ArrayList<Event>();
for(Event event : events) {
Event intercept1 = intercept(event);
if(intercept1 != null) {
interceptors.add(intercept1);
}
}
return interceptors;
}
public void close() {
// TODO Auto-generated method stub
}
public static class Builder implements Interceptor.Builder{
public void configure(Context context) {
// TODO Auto-generated method stub
}
public Interceptor build() {
// TODO Auto-generated method stub
return new ETLInterceptor();
}
}
}
拦截器实现时,需要继承一个公共接口,并实现其中的四个接口方法
静态类Builder是将jar放入flume的lib时,flume创建拦截器的方式
方法工具类
package com.myflume.interceptor;
import org.apache.commons.lang.NumberUtils;
public class LogUtils {
public static boolean valuateStart(String log) {
if(log == null) {
return false;
}
if (!log.trim().startsWith("{") || !log.trim().endsWith("}")) {
return false;
}
//格式 json
return true;
}
public static boolean valuateEvent(String log) {
// TODO Auto-generated method stub
if(log == null) {
return false;
}
//格式 时间 | json
String[] logContents = log.split("\\|");
if(logContents.length != 2) {
return false;
}
if(logContents[0].length() != 13) {
return false;
}
if (logContents[0].length() != 13 || !NumberUtils.isDigits(logContents[0])) {
return false;
}
if (!logContents[1].trim().startsWith("{") || !logContents[1].trim().endsWith("}")) {
return false;
}
return true;
}
}

浙公网安备 33010602011771号