logback输出json格式日志（包括mdc）发送到kafka
1，pom.xml
<!-- kafka --> <dependency> <groupId>com.github.danielwegener</groupId> <artifactId>logback-kafka-appender</artifactId> <version>0.2.0-RC1</version> <scope>runtime</scope> </dependency> <!-- logback--> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-core</artifactId> </dependency> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <version>1.2.3</version> <scope>runtime</scope> </dependency> <dependency> <groupId>net.logstash.logback</groupId> <artifactId>logstash-logback-encoder</artifactId> <version>5.0</version> </dependency> <!-- other--> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>1.7.25</version> <scope>compile</scope> </dependency> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-access</artifactId> </dependency>
2, spring-logback.xml
<?xml version="1.0" encoding="UTF-8"?> <configuration debug="false" scan="true" scanPeriod="600000"> <!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径 --> <property name="LOG_HOME" value="/var/log" /> <contextName>${HOSTNAME}</contextName> <springProperty scope="context" name="appName" source="spring.application.name" /> <springProperty scope="context" name="ip" source="spring.cloud.client.ipAddress" /> <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 --> <property name="CONSOLE_LOG_PATTERN" value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n" /> <!-- <logger name="org.springframework.web" level="DEBUG" /> --> <!-- show parameters for hibernate sql 专为 Hibernate 定制 --> <!--<logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="TRACE" /> --> <!--<logger name="org.hibernate.type.descriptor.sql.BasicExtractor" level="DEBUG" /> --> <!--<logger name="org.hibernate.engine.QueryParameters" level="DEBUG" /> --> <!--<logger name="org.hibernate.engine.query.HQLQueryPlan" level="DEBUG" /> --> <!-- <logger name="org.hibernate.SQL" level="DEBUG" /> --> <logger name="logging.level.com.italktv.platform" level="info" /> <!-- 控制台输出 --> <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> <encoder> <pattern>${CONSOLE_LOG_PATTERN}</pattern> <charset>utf-8</charset> </encoder> <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> <level>debug</level> </filter> </appender> <!-- 按照每天生成日志文件 --> <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <!-- 正在记录的日志文件的路径及文件名 --> <file>${LOG_HOME}/bigdata/data-api.log</file> <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> <!--日志文件输出的文件名 --> <FileNamePattern>${LOG_HOME}/bigdata/data-api.%d{yyyy-MM-dd}.%i.log </FileNamePattern> <!--日志文件保留天数 --> <MaxHistory>30</MaxHistory> <maxFileSize>1MB</maxFileSize> <totalSizeCap>10MB</totalSizeCap> </rollingPolicy> <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder"> <providers> <timestamp> <timeZone>UTC</timeZone> </timestamp> <pattern> <pattern> { "level": "%level", "trace": "%X{X-B3-TraceId:-}", "requestId": "%X{requestId}", "remoteIp": "%X{remoteIp}", "span": "%X{X-B3-SpanId:-}", "parent": "%X{X-B3-ParentSpanId:-}", "thread": "%thread", "class": "%logger{40}", "message": "%message", "stack_trace": "%exception{10}" } </pattern> </pattern> </providers> </encoder> <!--日志文件最大的大小 <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> <MaxFileSize>10KB</MaxFileSize> </triggeringPolicy> --> </appender> <!-- pom dependency <dependency> <groupId>com.github.danielwegener</groupId> <artifactId>logback-kafka-appender</artifactId> <version>0.1.0</version> <scope>runtime</scope> </dependency> <dependency> <groupId>net.logstash.logback</groupId> <artifactId>logstash-logback-encoder</artifactId> <version>4.11</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>1.7.25</version> <scope>compile</scope> </dependency> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-core</artifactId> <version>1.1.11</version> </dependency> <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <version>1.1.11</version> </dependency> --> <!-- other appender <appender name="kafkaAppenderAnotherEncode" class="com.github.danielwegener.logback.kafka.KafkaAppender"> <encoder class="com.github.danielwegener.logback.kafka.encoding.PatternLayoutKafkaMessageEncoder"> <layout class="net.logstash.logback.layout.LogstashLayout"> <includeMdc>true</includeMdc> <includeContext>true</includeContext> <includeCallerData>true</includeCallerData> <customFields>{"system":"test"}</customFields> <fieldNames class="net.logstash.logback.fieldnames.ShortenedFieldNames" /> </layout> </encoder> <topic>tv_server_logstash_log</topic> <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" /> <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" /> <producerConfig>bootstrap.servers=211.100.75.227:9092</producerConfig> <producerConfig>acks=0</producerConfig> <producerConfig>linger.ms=1000</producerConfig> <producerConfig>block.on.buffer.full=false</producerConfig> <appender-ref ref="STDOUT" /> </appender> --> <!-- https://www.cnblogs.com/maxzhang1985/p/9522507.html https://logback.qos.ch/manual/layouts.html --> <appender name="kafkaAppender" class="com.github.danielwegener.logback.kafka.KafkaAppender"> <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder"> <providers> <mdc /> <context /> <timestamp> <timeZone>UTC</timeZone> </timestamp> <pattern> <pattern> { "level": "%level", "trace": "%X{X-B3-TraceId:-}", "span": "%X{X-B3-SpanId:-}", "parent": "%X{X-B3-ParentSpanId:-}", "thread": "%thread", "class": "%logger{40}", "message": "%message", "stack_trace": "%exception{10}" } </pattern> </pattern> </providers> </encoder> <topic>tv_server_logstash_log</topic> <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" /> <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" /> <producerConfig>bootstrap.servers=127.0.0.1:9092</producerConfig> <!-- don't wait for a broker to ack the reception of a batch. --> <producerConfig>acks=0</producerConfig> <!-- wait up to 1000ms and collect log messages before sending them as a batch --> <producerConfig>linger.ms=1000</producerConfig> <!-- even if the producer buffer runs full, do not block the application but start to drop messages --> <!--<producerConfig>max.block.ms=0</producerConfig> --> <producerConfig>block.on.buffer.full=false</producerConfig> <!-- kafka连接失败后，使用下面配置进行日志输出 --> <appender-ref ref="STDOUT" /> </appender> <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender"> <appender-ref ref="kafkaAppender" /> </appender> <!-- 日志输出级别 --> <root level="INFO"> <!-- 生产上不输出stdout log --> <appender-ref ref="STDOUT" /> <!-- <appender-ref ref="FILE" /> --> <!-- <appender-ref ref="kafkaAppender" /> --> <appender-ref ref="ASYNC" /> </root> </configuration>
3, 添加一个mdc在logback
import java.util.UUID; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.ModelAndView; @Component public class LogInterceptor implements HandlerInterceptor { private final static String REQUEST_ID = "requestId"; private static final Logger LOGGER = LoggerFactory.getLogger(LogInterceptor.class); @Override public boolean preHandle(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o) throws Exception { String xForwardedForHeader = httpServletRequest.getHeader("X-Forwarded-For"); String remoteIp = httpServletRequest.getRemoteAddr(); String uuid = UUID.randomUUID().toString(); LOGGER.info("put requestId ({}) to logger", uuid); LOGGER.info("request id:{}, client ip:{}, X-Forwarded-For:{}", uuid, remoteIp, xForwardedForHeader); MDC.put(REQUEST_ID, uuid); MDC.put("remoteIp", remoteIp); return true; } @Override public void postHandle(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, ModelAndView modelAndView) throws Exception { String uuid = MDC.get(REQUEST_ID); LOGGER.info("remove requestId ({}) from logger", uuid); MDC.remove(REQUEST_ID); } @Override public void afterCompletion(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, Exception e) throws Exception { } }
4，添加切面 intercept
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; @Configuration public class WebMvcConfigurer extends WebMvcConfigurerAdapter { @Autowired private LogInterceptor logInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { registry.addInterceptor(logInterceptor); super.addInterceptors(registry); } }
参考：
logback 手册：https://logback.qos.ch/manual/layouts.html
http://www.importnew.com/28541.html
https://www.jianshu.com/p/a26da0c55255
https://blog.csdn.net/Soinice/article/details/84033382
https://examples.javacodegeeks.com/enterprise-java/logback/logback-kafka-appender-example/
http://stevetarver.github.io/2016/04/20/whole-product-logging.html 讲解详细
