flume 拦截器使用

 
 
拦截器使用:
       
使用目的:
            Flume中的拦截器(interceptor),用户Source读取events发送到Sink的时候,在events header中加入一些有用的信息,或者对events的内容进行过滤,完成初步的数据清洗。
 
拦截器的使用方式:
            通过配置文件
            如果使用了flume集群,需要将该配置文件,发送到每个flume的conf目录下
 
拦截器加载方式:
            拦截器打包之后,放入Flume的lib文件夹下面,flume会根据配置文件自动调用之(只需要单独包,不需要将依赖的包上传)。
            如果使用了flume集群,需要将该jar包,发送到每个flume的lib目录下
 
 
附:拦截器使用的配置文件
 
a1.sources=r1
a1.channels=c1 c2
 
# configure source
a1.sources.r1.type = TAILDIR
a1.sources.r1.positionFile = /opt/module/flume/test/log_position.json
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1 = /tmp/logs/app.+
a1.sources.r1.fileHeader = true                                  #flume数据的header将来会用到,如写header数据
a1.sources.r1.channels = c1 c2
 
#interceptor
a1.sources.r1.interceptors =  i1 i2
a1.sources.r1.interceptors.i1.type = com.flume.interceptor.ETLInterceptor$Builder     #指定拦截器,$Builder表示加载方式
a1.sources.r1.interceptors.i2.type = com.flume.interceptor.TypeInterceptor$Builder
 
a1.sources.r1.selector.type = multiplexing                #根据sourcer和sinke关系发送,
a1.sources.r1.selector.header = topic                         #以下三行,组成关系
a1.sources.r1.selector.mapping.topic_start = c1       # c1接收的数据是 header中 {"topic":  "topic_start"}
a1.sources.r1.selector.mapping.topic_event = c2      # c2接收的数据是header中 {"topic":  "topic_event"}
 
# configure channel
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.channels.c1.kafka.topic = topic_start
a1.channels.c1.parseAsFlumeEvent = false                          #不在数据的前面,加前缀。如果数据为data,加后就是 {"topic_start" : data}
a1.channels.c1.kafka.consumer.group.id = flume-consumer
 
a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c2.kafka.bootstrap.servers = hadoop102:9092,hadoop103:9092,hadoop104:9092
a1.channels.c2.kafka.topic = topic_event
a1.channels.c2.parseAsFlumeEvent = false
a1.channels.c2.kafka.consumer.group.id = flume-consumer
 
 
拦截器实现:
package com.myflume;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import com.myflume.interceptor.LogUtils;
public class ETLInterceptor implements Interceptor {
       public void initialize() {
             // TODO Auto-generated method stub
             
       }
       public Event intercept(Event event) {
             // TODO Auto-generated method stub
             // 1- 获取数据
             // 单个Event处理
             byte[] body = event.getBody();
             String log = new String(body, Charset.forName("UTF-8"));
             
             // 2 校验  启动日志(JSON)
             if (log.contains("start")) {
                    
                    if(LogUtils.valuateStart(log)) {
                           return event;
                    }
                    
             }else {
                    if(LogUtils.valuateEvent(log)) {
                           return event;
                    }
             }
             
             return null;
       }
       public List<Event> intercept(List<Event> events) {
             // TODO Auto-generated method stub
             ArrayList<Event> interceptors = new ArrayList<Event>();
             
             for(Event event : events) {
                    Event intercept1 = intercept(event);
                    if(intercept1 != null) {
                           interceptors.add(intercept1);
                    }
             }
             return interceptors;
       }
       public void close() {
             // TODO Auto-generated method stub
             
       }
       public static class Builder implements Interceptor.Builder{
             public void configure(Context context) {
                    // TODO Auto-generated method stub
                    
             }
             public Interceptor build() {
                    // TODO Auto-generated method stub
                    return new ETLInterceptor();
             }
             
       }
       
}
        拦截器实现时,需要继承一个公共接口,并实现其中的四个接口方法
        静态类Builder是将jar放入flume的lib时,flume创建拦截器的方式
 
 
 
方法工具类
package com.myflume.interceptor;
import org.apache.commons.lang.NumberUtils;
public class LogUtils {
       public static boolean valuateStart(String log) {
             
             if(log == null) {
                    return false;
             }
             
             if (!log.trim().startsWith("{") || !log.trim().endsWith("}")) {
                    return false;
             }
             
             //格式 json
             return true;
       }
       public static boolean valuateEvent(String log) {
             // TODO Auto-generated method stub
             
             if(log == null) {
                    return false;
             }
             
             //格式 时间 | json
             String[] logContents = log.split("\\|");
             
             if(logContents.length != 2) {
                    return false;
             }
             
             if(logContents[0].length() != 13) {
                    return false;
             }
             
             if (logContents[0].length() != 13 || !NumberUtils.isDigits(logContents[0])) {
                    return false;
             }
             
             if (!logContents[1].trim().startsWith("{") || !logContents[1].trim().endsWith("}")) {
                    return false;
             }
             
             return true;
       }
}
 
 
 
 
posted @ 2020-12-19 22:17  dos_hello_world  阅读(603)  评论(0)    收藏  举报